In [ ]:
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds = plotResults(y_test, anomalyScoresAE, True)
model.reset_states()
2938/2938 [==============================] - 2s 539us/step
May 2021 Update: Support for TensorFlow 2. Note that the results in this notebook differ from those in the book because of randomness in the seed we have set here vs. the original seed for TensorFlow and numpy in the book. This is expected. I encourage you to experiment on your own by changing the seed for TensorFlow and numpy in this notebook.
Note for Mac users: You will need to use the "tensorflow_mac" Conda environment for this notebook instead of the "unsupervisedLearning" Conda environment. Please refer to the README on the GitHub repo for more.
From Hands-on Unsupervised Learning using Python - Chapter 8 - Hands-on Autoencoder
Only for users on Google Colab
# Define functions to connect to Google and change directories
def connectDrive():
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
def changeDirectory(path):
import os
original_path = os.getcwd()
os.chdir(path)
new_path = os.getcwd()
print("Original path: ",original_path)
print("New path: ",new_path)
# Connect to Google Drive
connectDrive()
# Change path
changeDirectory("/content/drive/My Drive/github/handson-unsupervised-learning/")
Mounted at /content/drive Original path: /content New path: /content/drive/My Drive/github/handson-unsupervised-learning
'''Main'''
import numpy as np
import pandas as pd
import os, time, re
import pickle, gzip
'''Data Viz'''
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns
color = sns.color_palette()
%matplotlib inline
'''Data Prep and Model Evaluation'''
from sklearn import preprocessing as pp
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import log_loss
from sklearn.metrics import precision_recall_curve, average_precision_score
from sklearn.metrics import roc_curve, auc, roc_auc_score
'''Algos'''
import lightgbm as lgb
'''TensorFlow and Keras'''
import tensorflow as tf
from tensorflow import keras
K = keras.backend
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Activation, Dense, Dropout
from tensorflow.keras.layers import BatchNormalization, Input, Lambda
from tensorflow.keras import regularizers
from tensorflow.keras.losses import mse, binary_crossentropy
import sys, sklearn
print(f'sklearn {sklearn.__version__}')
print(f'tensorflow {tf.__version__}')
print(f'keras {keras.__version__}')
print(f'numpy {np.__version__}')
sklearn 0.22.2.post1 tensorflow 2.4.1 keras 2.4.0 numpy 1.19.5
# To make the output stable across runs
tf.random.set_seed(42)
np.random.seed(42)
# Check use of GPU
if tf.test.gpu_device_name():
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
else:
print("Please install GPU version of TF, if GPU is available.")
Please install GPU version of TF, if GPU is available.
current_path = os.getcwd()
file = os.path.sep.join(['', 'datasets', 'credit_card_data', 'credit_card.csv'])
data = pd.read_csv(current_path + file)
dataX = data.copy().drop(['Class','Time'],axis=1)
dataY = data['Class'].copy()
X_train, X_test, y_train, y_test = train_test_split(
dataX, dataY, test_size=0.33,
random_state=2018, stratify=dataY)
# Suppress warnings
pd.set_option('mode.chained_assignment', None)
featuresToScale = dataX.columns
sX = pp.StandardScaler(copy=True, with_mean=True, with_std=True)
X_train.loc[:,featuresToScale] = sX.fit_transform(X_train.loc[:,featuresToScale])
X_test.loc[:,featuresToScale] = sX.transform(X_test.loc[:,featuresToScale])
X_train_AE, X_test_AE = X_train.copy(), X_test.copy()
def anomalyScores(originalDF, reducedDF):
loss = np.sum((np.array(originalDF) -
np.array(reducedDF))**2, axis=1)
loss = pd.Series(data=loss,index=originalDF.index)
loss = (loss-np.min(loss))/(np.max(loss)-np.min(loss))
return loss
def plotResults(trueLabels, anomalyScores, returnPreds = False):
preds = pd.concat([trueLabels, anomalyScores], axis=1)
preds.columns = ['trueLabel', 'anomalyScore']
precision, recall, thresholds = \
precision_recall_curve(preds['trueLabel'],
preds['anomalyScore'])
average_precision = average_precision_score(
preds['trueLabel'], preds['anomalyScore'])
plt.step(recall, precision, color='k', alpha=0.7, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.3, color='k')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('Precision-Recall curve: Average Precision = \
{0:0.2f}'.format(average_precision))
fpr, tpr, thresholds = roc_curve(preds['trueLabel'],
preds['anomalyScore'])
areaUnderROC = auc(fpr, tpr)
plt.figure()
plt.plot(fpr, tpr, color='r', lw=2, label='ROC curve')
plt.plot([0, 1], [0, 1], color='k', lw=2, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic: Area under the \
curve = {0:0.2f}'.format(areaUnderROC))
plt.legend(loc="lower right")
plt.show()
if returnPreds==True:
return preds, average_precision
tf.random.set_seed(42)
np.random.seed(42)
# Call neural network API
model = Sequential()
# Apply linear activation function to input layer
# Generate hidden layer with 29 nodes, the same as the input layer
model.add(Dense(units=29, activation='linear',input_dim=29))
# Apply linear activation function to hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2891 - accuracy: 0.7103 - val_loss: 0.0011 - val_accuracy: 0.9836 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0014 - accuracy: 0.9889 - val_loss: 1.4438e-05 - val_accuracy: 0.9983 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.7539e-04 - accuracy: 0.9937 - val_loss: 2.6702e-04 - val_accuracy: 0.9944 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.2259e-04 - accuracy: 0.9948 - val_loss: 6.7056e-06 - val_accuracy: 0.9993 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.4883e-04 - accuracy: 0.9931 - val_loss: 5.0246e-05 - val_accuracy: 0.9969 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.7656e-04 - accuracy: 0.9964 - val_loss: 0.0012 - val_accuracy: 0.9860 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.4220e-04 - accuracy: 0.9954 - val_loss: 7.7994e-04 - val_accuracy: 0.9898 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9926 - val_loss: 7.1407e-06 - val_accuracy: 0.9988 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9938 - val_loss: 0.0021 - val_accuracy: 0.9864 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.1810e-04 - accuracy: 0.9934 - val_loss: 0.0028 - val_accuracy: 0.9727
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds = plotResults(y_test, anomalyScoresAE, True)
model.reset_states()
2938/2938 [==============================] - 2s 539us/step
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply linear activation function to input layer
# Generate hidden layer with 29 nodes, the same as the input layer
model.add(Dense(units=29, activation='linear',input_dim=29))
# Apply linear activation function to hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f'Mean average precision over 10 runs: {np.mean(test_scores)}')
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3308 - accuracy: 0.6682 - val_loss: 0.0019 - val_accuracy: 0.9778 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0014 - accuracy: 0.9886 - val_loss: 1.6041e-06 - val_accuracy: 0.9994 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.9293e-04 - accuracy: 0.9940 - val_loss: 7.7672e-04 - val_accuracy: 0.9915 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4527e-04 - accuracy: 0.9943 - val_loss: 2.3835e-06 - val_accuracy: 0.9993 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.0941e-04 - accuracy: 0.9948 - val_loss: 1.2261e-04 - val_accuracy: 0.9946 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.8485e-04 - accuracy: 0.9961 - val_loss: 7.8218e-04 - val_accuracy: 0.9874 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.5350e-04 - accuracy: 0.9958 - val_loss: 0.0021 - val_accuracy: 0.9826 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9903 - val_loss: 8.8468e-06 - val_accuracy: 0.9987 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9938 - val_loss: 0.0028 - val_accuracy: 0.9854 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.7426e-04 - accuracy: 0.9937 - val_loss: 0.0018 - val_accuracy: 0.9800 2938/2938 [==============================] - 2s 525us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2790 - accuracy: 0.7099 - val_loss: 0.0017 - val_accuracy: 0.9748 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9891 - val_loss: 2.8647e-04 - val_accuracy: 0.9948 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.8624e-04 - accuracy: 0.9943 - val_loss: 7.2143e-04 - val_accuracy: 0.9897 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.8828e-04 - accuracy: 0.9955 - val_loss: 3.5368e-06 - val_accuracy: 0.9994 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.7223e-04 - accuracy: 0.9943 - val_loss: 9.0208e-05 - val_accuracy: 0.9959 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.4415e-04 - accuracy: 0.9968 - val_loss: 6.9744e-04 - val_accuracy: 0.9898 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.2752e-04 - accuracy: 0.9959 - val_loss: 0.0022 - val_accuracy: 0.9872 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0026 - accuracy: 0.9909 - val_loss: 5.5084e-05 - val_accuracy: 0.9971 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9939 - val_loss: 0.0019 - val_accuracy: 0.9880 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.5812e-04 - accuracy: 0.9934 - val_loss: 0.0028 - val_accuracy: 0.9727 2938/2938 [==============================] - 2s 550us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2838 - accuracy: 0.7271 - val_loss: 0.0022 - val_accuracy: 0.9832 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0030 - accuracy: 0.9827 - val_loss: 4.5142e-04 - val_accuracy: 0.9929 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.6377e-04 - accuracy: 0.9923 - val_loss: 7.0631e-04 - val_accuracy: 0.9929 Epoch 4/10 5964/5964 [==============================] - 11s 2ms/step - loss: 3.5409e-04 - accuracy: 0.9956 - val_loss: 0.0011 - val_accuracy: 0.9924 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.9101e-04 - accuracy: 0.9920 - val_loss: 3.3091e-05 - val_accuracy: 0.9970 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.9661e-04 - accuracy: 0.9965 - val_loss: 5.1622e-04 - val_accuracy: 0.9892 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.4047e-04 - accuracy: 0.9956 - val_loss: 0.0020 - val_accuracy: 0.9859 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9917 - val_loss: 9.0804e-06 - val_accuracy: 0.9987 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.8690e-04 - accuracy: 0.9944 - val_loss: 4.9165e-04 - val_accuracy: 0.9931 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.6118e-04 - accuracy: 0.9941 - val_loss: 0.0035 - val_accuracy: 0.9720 2938/2938 [==============================] - 2s 527us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3062 - accuracy: 0.7099 - val_loss: 0.0011 - val_accuracy: 0.9803 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9918 - val_loss: 7.9635e-05 - val_accuracy: 0.9960 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 7.2936e-04 - accuracy: 0.9940 - val_loss: 8.3035e-04 - val_accuracy: 0.9911 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.8715e-04 - accuracy: 0.9941 - val_loss: 2.3968e-06 - val_accuracy: 0.9994 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.1669e-04 - accuracy: 0.9943 - val_loss: 7.1605e-05 - val_accuracy: 0.9960 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.7103e-04 - accuracy: 0.9968 - val_loss: 0.0012 - val_accuracy: 0.9832 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.7448e-04 - accuracy: 0.9951 - val_loss: 0.0012 - val_accuracy: 0.9829 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9934 - val_loss: 2.0509e-05 - val_accuracy: 0.9980 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9943 - val_loss: 8.7244e-04 - val_accuracy: 0.9902 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.3660e-04 - accuracy: 0.9948 - val_loss: 0.0017 - val_accuracy: 0.9804 2938/2938 [==============================] - 2s 512us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3046 - accuracy: 0.6929 - val_loss: 0.0017 - val_accuracy: 0.9824 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9866 - val_loss: 4.5951e-06 - val_accuracy: 0.9989 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.2834e-04 - accuracy: 0.9943 - val_loss: 4.2257e-04 - val_accuracy: 0.9913 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 3.0219e-04 - accuracy: 0.9953 - val_loss: 2.6139e-06 - val_accuracy: 0.9995 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.7480e-04 - accuracy: 0.9935 - val_loss: 1.1055e-04 - val_accuracy: 0.9953 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.1186e-04 - accuracy: 0.9969 - val_loss: 8.6972e-04 - val_accuracy: 0.9880 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 2.9375e-04 - accuracy: 0.9958 - val_loss: 2.5760e-04 - val_accuracy: 0.9950 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9923 - val_loss: 4.5907e-05 - val_accuracy: 0.9976 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.1548e-04 - accuracy: 0.9946 - val_loss: 0.0017 - val_accuracy: 0.9857 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.1128e-04 - accuracy: 0.9931 - val_loss: 0.0022 - val_accuracy: 0.9772 2938/2938 [==============================] - 2s 574us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2864 - accuracy: 0.7209 - val_loss: 0.0023 - val_accuracy: 0.9791 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9823 - val_loss: 5.3693e-04 - val_accuracy: 0.9903 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.4016e-04 - accuracy: 0.9920 - val_loss: 7.0501e-04 - val_accuracy: 0.9907 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 3.4446e-04 - accuracy: 0.9951 - val_loss: 1.1085e-06 - val_accuracy: 0.9996 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 7.7064e-04 - accuracy: 0.9937 - val_loss: 4.0008e-05 - val_accuracy: 0.9973 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 2.6982e-04 - accuracy: 0.9968 - val_loss: 0.0014 - val_accuracy: 0.9852 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 3.1062e-04 - accuracy: 0.9954 - val_loss: 0.0021 - val_accuracy: 0.9839 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0020 - accuracy: 0.9914 - val_loss: 1.0208e-05 - val_accuracy: 0.9987 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 9.2415e-04 - accuracy: 0.9937 - val_loss: 0.0018 - val_accuracy: 0.9887 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.8325e-04 - accuracy: 0.9933 - val_loss: 0.0020 - val_accuracy: 0.9777 2938/2938 [==============================] - 2s 529us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2835 - accuracy: 0.7168 - val_loss: 0.0016 - val_accuracy: 0.9818 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9890 - val_loss: 1.2850e-04 - val_accuracy: 0.9956 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9922 - val_loss: 8.5453e-04 - val_accuracy: 0.9898 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.5226e-04 - accuracy: 0.9954 - val_loss: 8.6625e-07 - val_accuracy: 0.9996 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.0864e-04 - accuracy: 0.9951 - val_loss: 2.8528e-05 - val_accuracy: 0.9979 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.0082e-04 - accuracy: 0.9969 - val_loss: 9.7033e-04 - val_accuracy: 0.9865 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.7527e-04 - accuracy: 0.9955 - val_loss: 0.0013 - val_accuracy: 0.9893 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9917 - val_loss: 1.0045e-05 - val_accuracy: 0.9986 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.6767e-04 - accuracy: 0.9933 - val_loss: 0.0015 - val_accuracy: 0.9894 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.1410e-04 - accuracy: 0.9941 - val_loss: 0.0032 - val_accuracy: 0.9726 2938/2938 [==============================] - 2s 535us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2762 - accuracy: 0.7164 - val_loss: 0.0021 - val_accuracy: 0.9776 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9857 - val_loss: 6.1264e-04 - val_accuracy: 0.9897 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0011 - accuracy: 0.9922 - val_loss: 3.0597e-04 - val_accuracy: 0.9933 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.7408e-04 - accuracy: 0.9952 - val_loss: 4.0847e-06 - val_accuracy: 0.9994 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.1425e-04 - accuracy: 0.9938 - val_loss: 2.3426e-04 - val_accuracy: 0.9941 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.6960e-04 - accuracy: 0.9970 - val_loss: 0.0012 - val_accuracy: 0.9826 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.0239e-04 - accuracy: 0.9952 - val_loss: 9.8072e-04 - val_accuracy: 0.9903 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9910 - val_loss: 4.5014e-06 - val_accuracy: 0.9991 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 9.4009e-04 - accuracy: 0.9937 - val_loss: 6.8393e-04 - val_accuracy: 0.9929 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.6423e-04 - accuracy: 0.9936 - val_loss: 0.0042 - val_accuracy: 0.9703 2938/2938 [==============================] - 2s 536us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2753 - accuracy: 0.7154 - val_loss: 0.0015 - val_accuracy: 0.9747 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9889 - val_loss: 1.7359e-04 - val_accuracy: 0.9939 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.4449e-04 - accuracy: 0.9939 - val_loss: 5.6167e-04 - val_accuracy: 0.9902 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.4783e-04 - accuracy: 0.9931 - val_loss: 4.2933e-06 - val_accuracy: 0.9993 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.5900e-04 - accuracy: 0.9935 - val_loss: 4.8684e-05 - val_accuracy: 0.9971 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.3541e-04 - accuracy: 0.9966 - val_loss: 0.0013 - val_accuracy: 0.9858 Epoch 7/10 5964/5964 [==============================] - 11s 2ms/step - loss: 2.8602e-04 - accuracy: 0.9956 - val_loss: 0.0023 - val_accuracy: 0.9857 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9929 - val_loss: 8.3821e-06 - val_accuracy: 0.9987 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9932 - val_loss: 0.0056 - val_accuracy: 0.9770 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.4442e-04 - accuracy: 0.9929 - val_loss: 0.0016 - val_accuracy: 0.9810 2938/2938 [==============================] - 2s 554us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2779 - accuracy: 0.7227 - val_loss: 0.0020 - val_accuracy: 0.9784 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9876 - val_loss: 3.9276e-05 - val_accuracy: 0.9974 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.3892e-04 - accuracy: 0.9946 - val_loss: 5.2945e-04 - val_accuracy: 0.9918 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.5143e-04 - accuracy: 0.9939 - val_loss: 2.3280e-04 - val_accuracy: 0.9966 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.2027e-04 - accuracy: 0.9925 - val_loss: 3.1208e-04 - val_accuracy: 0.9935 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 2.1241e-04 - accuracy: 0.9969 - val_loss: 0.0015 - val_accuracy: 0.9842 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 4.1192e-04 - accuracy: 0.9950 - val_loss: 0.0015 - val_accuracy: 0.9863 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9925 - val_loss: 4.9357e-05 - val_accuracy: 0.9970 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0022 - accuracy: 0.9938 - val_loss: 0.0023 - val_accuracy: 0.9855 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.8633e-04 - accuracy: 0.9932 - val_loss: 0.0026 - val_accuracy: 0.9746 2938/2938 [==============================] - 2s 506us/step
Mean average precision over 10 runs: 0.02877491279565572
[0.0372, 0.0166, 0.069, 0.0213, 0.0234, 0.017, 0.0195, 0.0263, 0.0247, 0.0328]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0288 Coefficient of variation over 10 runs: 0.5141
[0.0372, 0.0166, 0.069, 0.0213, 0.0234, 0.017, 0.0195, 0.0263, 0.0247, 0.0328]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply linear activation function to input layer
# Generate hidden layer with 20 nodes
model.add(Dense(units=20, activation='linear',input_dim=29))
# Apply linear activation function to hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5200 - accuracy: 0.4822 - val_loss: 0.2780 - val_accuracy: 0.6435 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2821 - accuracy: 0.6348 - val_loss: 0.2753 - val_accuracy: 0.6338 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2869 - accuracy: 0.6418 - val_loss: 0.2740 - val_accuracy: 0.6413 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6485 - val_loss: 0.2744 - val_accuracy: 0.6533 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2784 - accuracy: 0.6443 - val_loss: 0.2732 - val_accuracy: 0.6609 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2679 - accuracy: 0.6644 - val_loss: 0.2735 - val_accuracy: 0.6584 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2713 - accuracy: 0.6646 - val_loss: 0.2736 - val_accuracy: 0.6645 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2903 - accuracy: 0.6659 - val_loss: 0.2737 - val_accuracy: 0.6753 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6750 - val_loss: 0.2739 - val_accuracy: 0.6812 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2689 - accuracy: 0.6813 - val_loss: 0.2739 - val_accuracy: 0.6793 2938/2938 [==============================] - 2s 520us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5172 - accuracy: 0.4620 - val_loss: 0.2778 - val_accuracy: 0.6169 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2805 - accuracy: 0.6176 - val_loss: 0.2752 - val_accuracy: 0.6392 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2830 - accuracy: 0.6379 - val_loss: 0.2740 - val_accuracy: 0.6431 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2721 - accuracy: 0.6456 - val_loss: 0.2746 - val_accuracy: 0.6387 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2786 - accuracy: 0.6427 - val_loss: 0.2733 - val_accuracy: 0.6491 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2687 - accuracy: 0.6451 - val_loss: 0.2738 - val_accuracy: 0.6531 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2704 - accuracy: 0.6534 - val_loss: 0.2741 - val_accuracy: 0.6577 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2901 - accuracy: 0.6610 - val_loss: 0.2739 - val_accuracy: 0.6585 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2988 - accuracy: 0.6594 - val_loss: 0.2744 - val_accuracy: 0.6629 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2692 - accuracy: 0.6656 - val_loss: 0.2734 - val_accuracy: 0.6599 2938/2938 [==============================] - 2s 517us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5168 - accuracy: 0.4578 - val_loss: 0.2791 - val_accuracy: 0.6320 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2830 - accuracy: 0.6350 - val_loss: 0.2757 - val_accuracy: 0.6553 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2862 - accuracy: 0.6636 - val_loss: 0.2743 - val_accuracy: 0.6696 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2724 - accuracy: 0.6649 - val_loss: 0.2745 - val_accuracy: 0.6668 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2800 - accuracy: 0.6628 - val_loss: 0.2736 - val_accuracy: 0.6672 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2686 - accuracy: 0.6587 - val_loss: 0.2739 - val_accuracy: 0.6647 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2717 - accuracy: 0.6664 - val_loss: 0.2742 - val_accuracy: 0.6625 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2919 - accuracy: 0.6657 - val_loss: 0.2740 - val_accuracy: 0.6645 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2989 - accuracy: 0.6670 - val_loss: 0.2744 - val_accuracy: 0.6674 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2696 - accuracy: 0.6688 - val_loss: 0.2736 - val_accuracy: 0.6679 2938/2938 [==============================] - 2s 510us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5146 - accuracy: 0.4445 - val_loss: 0.2784 - val_accuracy: 0.6037 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2818 - accuracy: 0.6062 - val_loss: 0.2756 - val_accuracy: 0.6261 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2882 - accuracy: 0.6349 - val_loss: 0.2741 - val_accuracy: 0.6378 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2726 - accuracy: 0.6464 - val_loss: 0.2744 - val_accuracy: 0.6394 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2783 - accuracy: 0.6344 - val_loss: 0.2733 - val_accuracy: 0.6414 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2692 - accuracy: 0.6362 - val_loss: 0.2740 - val_accuracy: 0.6414 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2712 - accuracy: 0.6365 - val_loss: 0.2743 - val_accuracy: 0.6443 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2914 - accuracy: 0.6492 - val_loss: 0.2737 - val_accuracy: 0.6506 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2972 - accuracy: 0.6476 - val_loss: 0.2742 - val_accuracy: 0.6539 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2697 - accuracy: 0.6560 - val_loss: 0.2738 - val_accuracy: 0.6573 2938/2938 [==============================] - 2s 512us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5362 - accuracy: 0.4355 - val_loss: 0.2780 - val_accuracy: 0.5982 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2817 - accuracy: 0.6054 - val_loss: 0.2753 - val_accuracy: 0.6117 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2835 - accuracy: 0.6164 - val_loss: 0.2743 - val_accuracy: 0.6300 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2733 - accuracy: 0.6422 - val_loss: 0.2748 - val_accuracy: 0.6354 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2795 - accuracy: 0.6332 - val_loss: 0.2736 - val_accuracy: 0.6428 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2690 - accuracy: 0.6410 - val_loss: 0.2740 - val_accuracy: 0.6416 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6424 - val_loss: 0.2742 - val_accuracy: 0.6474 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2899 - accuracy: 0.6492 - val_loss: 0.2741 - val_accuracy: 0.6484 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2973 - accuracy: 0.6503 - val_loss: 0.2745 - val_accuracy: 0.6548 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2696 - accuracy: 0.6630 - val_loss: 0.2736 - val_accuracy: 0.6534 2938/2938 [==============================] - 2s 514us/step
Epoch 1/10 5964/5964 [==============================] - 12s 2ms/step - loss: 0.5290 - accuracy: 0.4867 - val_loss: 0.2791 - val_accuracy: 0.6367 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2820 - accuracy: 0.6318 - val_loss: 0.2761 - val_accuracy: 0.6496 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2856 - accuracy: 0.6561 - val_loss: 0.2749 - val_accuracy: 0.6517 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2739 - accuracy: 0.6584 - val_loss: 0.2748 - val_accuracy: 0.6539 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2799 - accuracy: 0.6559 - val_loss: 0.2735 - val_accuracy: 0.6597 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2685 - accuracy: 0.6561 - val_loss: 0.2741 - val_accuracy: 0.6692 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2711 - accuracy: 0.6701 - val_loss: 0.2742 - val_accuracy: 0.6809 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2908 - accuracy: 0.6835 - val_loss: 0.2740 - val_accuracy: 0.6887 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6870 - val_loss: 0.2743 - val_accuracy: 0.6915 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2698 - accuracy: 0.6941 - val_loss: 0.2736 - val_accuracy: 0.6894 2938/2938 [==============================] - 2s 513us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5167 - accuracy: 0.4460 - val_loss: 0.2788 - val_accuracy: 0.5749 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2816 - accuracy: 0.5687 - val_loss: 0.2759 - val_accuracy: 0.5731 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2859 - accuracy: 0.5806 - val_loss: 0.2746 - val_accuracy: 0.5731 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2740 - accuracy: 0.5857 - val_loss: 0.2749 - val_accuracy: 0.5876 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2802 - accuracy: 0.5914 - val_loss: 0.2738 - val_accuracy: 0.6063 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2701 - accuracy: 0.6020 - val_loss: 0.2744 - val_accuracy: 0.6095 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2712 - accuracy: 0.6125 - val_loss: 0.2753 - val_accuracy: 0.6252 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2918 - accuracy: 0.6334 - val_loss: 0.2739 - val_accuracy: 0.6328 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2971 - accuracy: 0.6334 - val_loss: 0.2745 - val_accuracy: 0.6443 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2703 - accuracy: 0.6430 - val_loss: 0.2738 - val_accuracy: 0.6521 2938/2938 [==============================] - 2s 534us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5696 - accuracy: 0.4388 - val_loss: 0.2783 - val_accuracy: 0.6143 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2801 - accuracy: 0.6045 - val_loss: 0.2753 - val_accuracy: 0.6202 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2860 - accuracy: 0.6201 - val_loss: 0.2740 - val_accuracy: 0.6374 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2730 - accuracy: 0.6566 - val_loss: 0.2743 - val_accuracy: 0.6483 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2791 - accuracy: 0.6471 - val_loss: 0.2730 - val_accuracy: 0.6524 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2681 - accuracy: 0.6522 - val_loss: 0.2736 - val_accuracy: 0.6477 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2712 - accuracy: 0.6558 - val_loss: 0.2736 - val_accuracy: 0.6614 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2902 - accuracy: 0.6650 - val_loss: 0.2737 - val_accuracy: 0.6683 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2974 - accuracy: 0.6713 - val_loss: 0.2738 - val_accuracy: 0.6687 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2691 - accuracy: 0.6710 - val_loss: 0.2734 - val_accuracy: 0.6711 2938/2938 [==============================] - 2s 543us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5008 - accuracy: 0.4727 - val_loss: 0.2786 - val_accuracy: 0.6051 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2817 - accuracy: 0.6110 - val_loss: 0.2761 - val_accuracy: 0.6221 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2881 - accuracy: 0.6389 - val_loss: 0.2745 - val_accuracy: 0.6381 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2737 - accuracy: 0.6440 - val_loss: 0.2746 - val_accuracy: 0.6432 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2783 - accuracy: 0.6418 - val_loss: 0.2734 - val_accuracy: 0.6472 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2682 - accuracy: 0.6459 - val_loss: 0.2739 - val_accuracy: 0.6498 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6565 - val_loss: 0.2737 - val_accuracy: 0.6567 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2904 - accuracy: 0.6615 - val_loss: 0.2736 - val_accuracy: 0.6627 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6640 - val_loss: 0.2739 - val_accuracy: 0.6643 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2690 - accuracy: 0.6656 - val_loss: 0.2735 - val_accuracy: 0.6685 2938/2938 [==============================] - 2s 534us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5229 - accuracy: 0.4444 - val_loss: 0.2801 - val_accuracy: 0.5858 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2817 - accuracy: 0.5887 - val_loss: 0.2768 - val_accuracy: 0.6055 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2882 - accuracy: 0.6165 - val_loss: 0.2752 - val_accuracy: 0.6374 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2730 - accuracy: 0.6495 - val_loss: 0.2750 - val_accuracy: 0.6591 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2797 - accuracy: 0.6565 - val_loss: 0.2737 - val_accuracy: 0.6645 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2681 - accuracy: 0.6602 - val_loss: 0.2743 - val_accuracy: 0.6672 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2720 - accuracy: 0.6661 - val_loss: 0.2742 - val_accuracy: 0.6706 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2921 - accuracy: 0.6697 - val_loss: 0.2739 - val_accuracy: 0.6696 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.2989 - accuracy: 0.6734 - val_loss: 0.2748 - val_accuracy: 0.6710 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2693 - accuracy: 0.6708 - val_loss: 0.2736 - val_accuracy: 0.6626 2938/2938 [==============================] - 2s 515us/step
Mean average precision over 10 runs: 0.2752
[0.2799, 0.2837, 0.2649, 0.2822, 0.2744, 0.2674, 0.2816, 0.2711, 0.2755, 0.271]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.2752 Coefficient of variation over 10 runs: 0.0225
[0.2799, 0.2837, 0.2649, 0.2822, 0.2744, 0.2674, 0.2816, 0.2711, 0.2755, 0.271]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply linear activation function to input layer
# Generate hidden layer with 27 nodes
model.add(Dense(units=27, activation='linear',input_dim=29))
# Apply linear activation function to hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3209 - accuracy: 0.6425 - val_loss: 0.0365 - val_accuracy: 0.8872 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0368 - accuracy: 0.8912 - val_loss: 0.0373 - val_accuracy: 0.9247 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0474 - accuracy: 0.9257 - val_loss: 0.0374 - val_accuracy: 0.9512 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0393 - accuracy: 0.9540 - val_loss: 0.0377 - val_accuracy: 0.9516 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0382 - accuracy: 0.9545 - val_loss: 0.0363 - val_accuracy: 0.9615 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9592 - val_loss: 0.0376 - val_accuracy: 0.9510 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0352 - accuracy: 0.9583 - val_loss: 0.0382 - val_accuracy: 0.9532 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0547 - accuracy: 0.9486 - val_loss: 0.0371 - val_accuracy: 0.9637 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0572 - accuracy: 0.9560 - val_loss: 0.0404 - val_accuracy: 0.9560 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0329 - accuracy: 0.9620 - val_loss: 0.0384 - val_accuracy: 0.9529 2938/2938 [==============================] - 2s 512us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3188 - accuracy: 0.6564 - val_loss: 0.0373 - val_accuracy: 0.9131 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0402 - accuracy: 0.9055 - val_loss: 0.0358 - val_accuracy: 0.9328 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0408 - accuracy: 0.9228 - val_loss: 0.0358 - val_accuracy: 0.9536 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0353 - accuracy: 0.9527 - val_loss: 0.0368 - val_accuracy: 0.9379 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0368 - accuracy: 0.9439 - val_loss: 0.0356 - val_accuracy: 0.9412 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0357 - accuracy: 0.9494 - val_loss: 0.0362 - val_accuracy: 0.9445 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0359 - accuracy: 0.9500 - val_loss: 0.0375 - val_accuracy: 0.9495 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0493 - accuracy: 0.9458 - val_loss: 0.0388 - val_accuracy: 0.9507 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0530 - accuracy: 0.9493 - val_loss: 0.0386 - val_accuracy: 0.9557 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0329 - accuracy: 0.9614 - val_loss: 0.0380 - val_accuracy: 0.9582 2938/2938 [==============================] - 2s 522us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3392 - accuracy: 0.6583 - val_loss: 0.0374 - val_accuracy: 0.9221 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0401 - accuracy: 0.9374 - val_loss: 0.0365 - val_accuracy: 0.9505 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0455 - accuracy: 0.9475 - val_loss: 0.0371 - val_accuracy: 0.9579 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0379 - accuracy: 0.9606 - val_loss: 0.0380 - val_accuracy: 0.9532 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9549 - val_loss: 0.0364 - val_accuracy: 0.9642 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0360 - accuracy: 0.9613 - val_loss: 0.0372 - val_accuracy: 0.9576 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0348 - accuracy: 0.9600 - val_loss: 0.0367 - val_accuracy: 0.9604 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0546 - accuracy: 0.9508 - val_loss: 0.0371 - val_accuracy: 0.9638 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0580 - accuracy: 0.9594 - val_loss: 0.0387 - val_accuracy: 0.9602 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9637 - val_loss: 0.0379 - val_accuracy: 0.9580 2938/2938 [==============================] - 2s 536us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3200 - accuracy: 0.6504 - val_loss: 0.0380 - val_accuracy: 0.9415 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.9543 - val_loss: 0.0369 - val_accuracy: 0.9636 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0482 - accuracy: 0.9591 - val_loss: 0.0378 - val_accuracy: 0.9659 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0379 - accuracy: 0.9673 - val_loss: 0.0384 - val_accuracy: 0.9575 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9605 - val_loss: 0.0365 - val_accuracy: 0.9669 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9614 - val_loss: 0.0383 - val_accuracy: 0.9500 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0350 - accuracy: 0.9600 - val_loss: 0.0369 - val_accuracy: 0.9590 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0541 - accuracy: 0.9519 - val_loss: 0.0373 - val_accuracy: 0.9633 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0582 - accuracy: 0.9596 - val_loss: 0.0389 - val_accuracy: 0.9595 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9628 - val_loss: 0.0378 - val_accuracy: 0.9550 2938/2938 [==============================] - 2s 511us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3337 - accuracy: 0.6521 - val_loss: 0.0375 - val_accuracy: 0.9155 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0404 - accuracy: 0.9305 - val_loss: 0.0369 - val_accuracy: 0.9474 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0467 - accuracy: 0.9467 - val_loss: 0.0375 - val_accuracy: 0.9534 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.9598 - val_loss: 0.0381 - val_accuracy: 0.9586 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0382 - accuracy: 0.9579 - val_loss: 0.0365 - val_accuracy: 0.9662 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0364 - accuracy: 0.9606 - val_loss: 0.0373 - val_accuracy: 0.9535 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0349 - accuracy: 0.9586 - val_loss: 0.0371 - val_accuracy: 0.9593 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0553 - accuracy: 0.9542 - val_loss: 0.0372 - val_accuracy: 0.9644 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0572 - accuracy: 0.9601 - val_loss: 0.0380 - val_accuracy: 0.9605 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0328 - accuracy: 0.9627 - val_loss: 0.0378 - val_accuracy: 0.9594 2938/2938 [==============================] - 2s 533us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3201 - accuracy: 0.6345 - val_loss: 0.0381 - val_accuracy: 0.8996 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8891 - val_loss: 0.0362 - val_accuracy: 0.9169 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0436 - accuracy: 0.9157 - val_loss: 0.0363 - val_accuracy: 0.9368 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.9385 - val_loss: 0.0373 - val_accuracy: 0.9443 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9469 - val_loss: 0.0361 - val_accuracy: 0.9582 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0366 - accuracy: 0.9551 - val_loss: 0.0372 - val_accuracy: 0.9520 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0356 - accuracy: 0.9593 - val_loss: 0.0372 - val_accuracy: 0.9604 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0533 - accuracy: 0.9568 - val_loss: 0.0371 - val_accuracy: 0.9634 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0558 - accuracy: 0.9589 - val_loss: 0.0379 - val_accuracy: 0.9588 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9624 - val_loss: 0.0383 - val_accuracy: 0.9574 2938/2938 [==============================] - 2s 559us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3314 - accuracy: 0.6595 - val_loss: 0.0382 - val_accuracy: 0.8808 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.8860 - val_loss: 0.0361 - val_accuracy: 0.9015 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.9021 - val_loss: 0.0360 - val_accuracy: 0.9038 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9135 - val_loss: 0.0370 - val_accuracy: 0.9152 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0376 - accuracy: 0.9180 - val_loss: 0.0362 - val_accuracy: 0.9280 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0372 - accuracy: 0.9300 - val_loss: 0.0375 - val_accuracy: 0.9215 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0351 - accuracy: 0.9334 - val_loss: 0.0371 - val_accuracy: 0.9464 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.9393 - val_loss: 0.0376 - val_accuracy: 0.9548 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0587 - accuracy: 0.9485 - val_loss: 0.0384 - val_accuracy: 0.9564 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0332 - accuracy: 0.9594 - val_loss: 0.0376 - val_accuracy: 0.9588 2938/2938 [==============================] - 2s 561us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3814 - accuracy: 0.6203 - val_loss: 0.0375 - val_accuracy: 0.9055 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0387 - accuracy: 0.9122 - val_loss: 0.0356 - val_accuracy: 0.9285 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0376 - accuracy: 0.9308 - val_loss: 0.0358 - val_accuracy: 0.9359 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9397 - val_loss: 0.0358 - val_accuracy: 0.9446 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0369 - accuracy: 0.9445 - val_loss: 0.0355 - val_accuracy: 0.9546 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0354 - accuracy: 0.9557 - val_loss: 0.0356 - val_accuracy: 0.9547 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9552 - val_loss: 0.0391 - val_accuracy: 0.9473 Epoch 8/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0397 - accuracy: 0.9573 - val_loss: 0.0359 - val_accuracy: 0.9548 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9596 - val_loss: 0.0369 - val_accuracy: 0.9472 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0356 - accuracy: 0.9472 - val_loss: 0.0371 - val_accuracy: 0.9515 2938/2938 [==============================] - 2s 529us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3032 - accuracy: 0.6632 - val_loss: 0.0369 - val_accuracy: 0.9214 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.9375 - val_loss: 0.0366 - val_accuracy: 0.9577 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9517 - val_loss: 0.0380 - val_accuracy: 0.9659 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0381 - accuracy: 0.9665 - val_loss: 0.0379 - val_accuracy: 0.9620 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9651 - val_loss: 0.0365 - val_accuracy: 0.9695 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0362 - accuracy: 0.9641 - val_loss: 0.0370 - val_accuracy: 0.9597 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0347 - accuracy: 0.9635 - val_loss: 0.0379 - val_accuracy: 0.9591 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0549 - accuracy: 0.9568 - val_loss: 0.0372 - val_accuracy: 0.9646 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0578 - accuracy: 0.9583 - val_loss: 0.0378 - val_accuracy: 0.9604 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0328 - accuracy: 0.9633 - val_loss: 0.0378 - val_accuracy: 0.9617 2938/2938 [==============================] - 2s 527us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3304 - accuracy: 0.6521 - val_loss: 0.0374 - val_accuracy: 0.9069 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0394 - accuracy: 0.9077 - val_loss: 0.0358 - val_accuracy: 0.9077 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0367 - accuracy: 0.9132 - val_loss: 0.0352 - val_accuracy: 0.9173 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0336 - accuracy: 0.9296 - val_loss: 0.0355 - val_accuracy: 0.9140 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0379 - accuracy: 0.9180 - val_loss: 0.0343 - val_accuracy: 0.9457 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0312 - accuracy: 0.9466 - val_loss: 0.0354 - val_accuracy: 0.9138 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0342 - accuracy: 0.9378 - val_loss: 0.0373 - val_accuracy: 0.9250 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0354 - accuracy: 0.9372 - val_loss: 0.0359 - val_accuracy: 0.9471 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9517 - val_loss: 0.0355 - val_accuracy: 0.9626 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9599 - val_loss: 0.0353 - val_accuracy: 0.9402 2938/2938 [==============================] - 2s 532us/step
Mean average precision over 10 runs: 0.0821
[0.0129, 0.02, 0.0131, 0.0191, 0.0129, 0.0149, 0.0138, 0.0135, 0.0164, 0.6844]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0821 Coefficient of variation over 10 runs: 2.4455
[0.0129, 0.02, 0.0131, 0.0191, 0.0129, 0.0149, 0.0138, 0.0135, 0.0164, 0.6844]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
tf.random.set_seed(42)
np.random.seed(42)
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply linear activation function to input layer
# Generate first hidden layer with 27 nodes
# Generate second hidden layer with 28 nodes
model.add(Dense(units=28, activation='linear',input_dim=29))
model.add(Dense(units=27, activation='linear'))
# Apply linear activation function to second hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.3168 - accuracy: 0.6229 - val_loss: 0.0369 - val_accuracy: 0.9341 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.9412 - val_loss: 0.0366 - val_accuracy: 0.9531 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9534 - val_loss: 0.0393 - val_accuracy: 0.9611 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9632 - val_loss: 0.0384 - val_accuracy: 0.9571 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9581 - val_loss: 0.0363 - val_accuracy: 0.9685 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0370 - accuracy: 0.9622 - val_loss: 0.0390 - val_accuracy: 0.9503 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0355 - accuracy: 0.9588 - val_loss: 0.0373 - val_accuracy: 0.9602 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0553 - accuracy: 0.9596 - val_loss: 0.0377 - val_accuracy: 0.9630 Epoch 9/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0577 - accuracy: 0.9580 - val_loss: 0.0391 - val_accuracy: 0.9571 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0333 - accuracy: 0.9597 - val_loss: 0.0384 - val_accuracy: 0.9564 2938/2938 [==============================] - 2s 523us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2993 - accuracy: 0.6666 - val_loss: 0.0388 - val_accuracy: 0.9175 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.9367 - val_loss: 0.0366 - val_accuracy: 0.9533 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0468 - accuracy: 0.9520 - val_loss: 0.0395 - val_accuracy: 0.9561 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0387 - accuracy: 0.9621 - val_loss: 0.0386 - val_accuracy: 0.9533 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.9561 - val_loss: 0.0364 - val_accuracy: 0.9656 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0364 - accuracy: 0.9621 - val_loss: 0.0375 - val_accuracy: 0.9534 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0352 - accuracy: 0.9600 - val_loss: 0.0391 - val_accuracy: 0.9532 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0561 - accuracy: 0.9560 - val_loss: 0.0375 - val_accuracy: 0.9627 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0586 - accuracy: 0.9573 - val_loss: 0.0386 - val_accuracy: 0.9609 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9615 - val_loss: 0.0389 - val_accuracy: 0.9510 2938/2938 [==============================] - 2s 560us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2930 - accuracy: 0.6313 - val_loss: 0.0420 - val_accuracy: 0.8860 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.8955 - val_loss: 0.0365 - val_accuracy: 0.9158 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0417 - accuracy: 0.9190 - val_loss: 0.0406 - val_accuracy: 0.9272 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0382 - accuracy: 0.9404 - val_loss: 0.0374 - val_accuracy: 0.9426 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0386 - accuracy: 0.9447 - val_loss: 0.0358 - val_accuracy: 0.9541 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9516 - val_loss: 0.0399 - val_accuracy: 0.9402 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9530 - val_loss: 0.0386 - val_accuracy: 0.9562 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0532 - accuracy: 0.9575 - val_loss: 0.0374 - val_accuracy: 0.9643 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0585 - accuracy: 0.9591 - val_loss: 0.0384 - val_accuracy: 0.9586 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0334 - accuracy: 0.9611 - val_loss: 0.0388 - val_accuracy: 0.9568 2938/2938 [==============================] - 2s 529us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3136 - accuracy: 0.6349 - val_loss: 0.0419 - val_accuracy: 0.9011 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0406 - accuracy: 0.9094 - val_loss: 0.0360 - val_accuracy: 0.9130 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.9085 - val_loss: 0.0373 - val_accuracy: 0.9132 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0386 - accuracy: 0.9240 - val_loss: 0.0377 - val_accuracy: 0.9314 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0390 - accuracy: 0.9342 - val_loss: 0.0356 - val_accuracy: 0.9473 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0367 - accuracy: 0.9470 - val_loss: 0.0370 - val_accuracy: 0.9432 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0363 - accuracy: 0.9501 - val_loss: 0.0378 - val_accuracy: 0.9524 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.9471 - val_loss: 0.0376 - val_accuracy: 0.9602 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0539 - accuracy: 0.9586 - val_loss: 0.0380 - val_accuracy: 0.9619 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0334 - accuracy: 0.9626 - val_loss: 0.0428 - val_accuracy: 0.9463 2938/2938 [==============================] - 2s 539us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2715 - accuracy: 0.6668 - val_loss: 0.0383 - val_accuracy: 0.9031 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.9190 - val_loss: 0.0365 - val_accuracy: 0.9249 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0439 - accuracy: 0.9223 - val_loss: 0.0394 - val_accuracy: 0.9306 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0393 - accuracy: 0.9384 - val_loss: 0.0376 - val_accuracy: 0.9408 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0383 - accuracy: 0.9393 - val_loss: 0.0365 - val_accuracy: 0.9477 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0369 - accuracy: 0.9454 - val_loss: 0.0380 - val_accuracy: 0.9413 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0347 - accuracy: 0.9481 - val_loss: 0.0385 - val_accuracy: 0.9539 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0547 - accuracy: 0.9499 - val_loss: 0.0380 - val_accuracy: 0.9618 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0596 - accuracy: 0.9588 - val_loss: 0.0391 - val_accuracy: 0.9599 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0331 - accuracy: 0.9621 - val_loss: 0.0386 - val_accuracy: 0.9554 2938/2938 [==============================] - 2s 560us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3235 - accuracy: 0.6349 - val_loss: 0.0383 - val_accuracy: 0.9317 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.9308 - val_loss: 0.0358 - val_accuracy: 0.9389 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9375 - val_loss: 0.0384 - val_accuracy: 0.9506 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.9519 - val_loss: 0.0369 - val_accuracy: 0.9583 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9575 - val_loss: 0.0359 - val_accuracy: 0.9681 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0370 - accuracy: 0.9634 - val_loss: 0.0383 - val_accuracy: 0.9514 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0365 - accuracy: 0.9616 - val_loss: 0.0380 - val_accuracy: 0.9581 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0521 - accuracy: 0.9608 - val_loss: 0.0373 - val_accuracy: 0.9654 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0521 - accuracy: 0.9653 - val_loss: 0.0386 - val_accuracy: 0.9611 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9618 - val_loss: 0.0395 - val_accuracy: 0.9499 2938/2938 [==============================] - 2s 527us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2988 - accuracy: 0.6713 - val_loss: 0.0373 - val_accuracy: 0.9022 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.9081 - val_loss: 0.0346 - val_accuracy: 0.9387 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0351 - accuracy: 0.9431 - val_loss: 0.0344 - val_accuracy: 0.9279 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0318 - accuracy: 0.9483 - val_loss: 0.0363 - val_accuracy: 0.9274 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.9297 - val_loss: 0.0348 - val_accuracy: 0.9482 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0325 - accuracy: 0.9494 - val_loss: 0.0384 - val_accuracy: 0.9256 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0348 - accuracy: 0.9461 - val_loss: 0.0366 - val_accuracy: 0.9236 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9260 - val_loss: 0.0365 - val_accuracy: 0.9445 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0410 - accuracy: 0.9481 - val_loss: 0.0357 - val_accuracy: 0.9665 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9615 - val_loss: 0.0380 - val_accuracy: 0.9391 2938/2938 [==============================] - 2s 619us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.3382 - accuracy: 0.6437 - val_loss: 0.0412 - val_accuracy: 0.9233 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9396 - val_loss: 0.0359 - val_accuracy: 0.9503 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.9504 - val_loss: 0.0369 - val_accuracy: 0.9564 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9573 - val_loss: 0.0373 - val_accuracy: 0.9547 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0395 - accuracy: 0.9568 - val_loss: 0.0360 - val_accuracy: 0.9696 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0373 - accuracy: 0.9630 - val_loss: 0.0369 - val_accuracy: 0.9613 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0361 - accuracy: 0.9620 - val_loss: 0.0398 - val_accuracy: 0.9581 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0544 - accuracy: 0.9579 - val_loss: 0.0372 - val_accuracy: 0.9651 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0564 - accuracy: 0.9607 - val_loss: 0.0381 - val_accuracy: 0.9614 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0334 - accuracy: 0.9619 - val_loss: 0.0385 - val_accuracy: 0.9565 2938/2938 [==============================] - 2s 520us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3120 - accuracy: 0.6217 - val_loss: 0.0394 - val_accuracy: 0.9011 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0409 - accuracy: 0.9200 - val_loss: 0.0371 - val_accuracy: 0.9392 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9359 - val_loss: 0.0379 - val_accuracy: 0.9548 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9559 - val_loss: 0.0376 - val_accuracy: 0.9558 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0386 - accuracy: 0.9535 - val_loss: 0.0362 - val_accuracy: 0.9637 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0370 - accuracy: 0.9580 - val_loss: 0.0399 - val_accuracy: 0.9411 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9567 - val_loss: 0.0399 - val_accuracy: 0.9519 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0559 - accuracy: 0.9515 - val_loss: 0.0373 - val_accuracy: 0.9644 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0575 - accuracy: 0.9603 - val_loss: 0.0390 - val_accuracy: 0.9566 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9600 - val_loss: 0.0426 - val_accuracy: 0.9401 2938/2938 [==============================] - 2s 519us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2936 - accuracy: 0.6258 - val_loss: 0.0375 - val_accuracy: 0.9018 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9130 - val_loss: 0.0354 - val_accuracy: 0.9232 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0374 - accuracy: 0.9222 - val_loss: 0.0366 - val_accuracy: 0.9320 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0387 - accuracy: 0.9328 - val_loss: 0.0368 - val_accuracy: 0.9403 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0392 - accuracy: 0.9385 - val_loss: 0.0356 - val_accuracy: 0.9538 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0363 - accuracy: 0.9489 - val_loss: 0.0393 - val_accuracy: 0.9390 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0368 - accuracy: 0.9514 - val_loss: 0.0398 - val_accuracy: 0.9532 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0518 - accuracy: 0.9553 - val_loss: 0.0374 - val_accuracy: 0.9608 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0542 - accuracy: 0.9621 - val_loss: 0.0379 - val_accuracy: 0.9599 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0332 - accuracy: 0.9614 - val_loss: 0.0393 - val_accuracy: 0.9512 2938/2938 [==============================] - 2s 530us/step
Mean average precision over 10 runs: 0.0872
[0.0153, 0.0167, 0.0145, 0.03, 0.0255, 0.0154, 0.6637, 0.0234, 0.045, 0.023]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0872 Coefficient of variation over 10 runs: 2.2049
[0.0153, 0.0167, 0.0145, 0.03, 0.0255, 0.0154, 0.6637, 0.0234, 0.045, 0.023]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply ReLu throughout
# Generate first hidden layer with 27 nodes
# Generate second hidden layer with 22 nodes
model.add(Dense(units=27, activation='relu',input_dim=29))
model.add(Dense(units=22, activation='relu'))
# Apply ReLu throughout
# Generate third hidden layer with 27 nodes
# Generate output layer with 29 nodes
model.add(Dense(units=27, activation='relu'))
model.add(Dense(units=29, activation='relu'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.7648 - accuracy: 0.4436 - val_loss: 0.6323 - val_accuracy: 0.7000 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.6283 - accuracy: 0.7122 - val_loss: 0.6233 - val_accuracy: 0.7208 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6270 - accuracy: 0.7276 - val_loss: 0.6217 - val_accuracy: 0.7225 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6165 - accuracy: 0.7355 - val_loss: 0.6220 - val_accuracy: 0.7257 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6273 - accuracy: 0.7426 - val_loss: 0.6184 - val_accuracy: 0.7403 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6122 - accuracy: 0.7490 - val_loss: 0.6184 - val_accuracy: 0.7430 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6160 - accuracy: 0.7485 - val_loss: 0.6208 - val_accuracy: 0.7273 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6416 - accuracy: 0.7164 - val_loss: 0.6181 - val_accuracy: 0.7472 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6336 - accuracy: 0.7441 - val_loss: 0.6161 - val_accuracy: 0.7523 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6147 - accuracy: 0.7552 - val_loss: 0.6186 - val_accuracy: 0.7385 2938/2938 [==============================] - 2s 525us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.7414 - accuracy: 0.4732 - val_loss: 0.5855 - val_accuracy: 0.7640 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5778 - accuracy: 0.7840 - val_loss: 0.5662 - val_accuracy: 0.8200 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5689 - accuracy: 0.8230 - val_loss: 0.5668 - val_accuracy: 0.8087 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5543 - accuracy: 0.8290 - val_loss: 0.5589 - val_accuracy: 0.8284 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5619 - accuracy: 0.8342 - val_loss: 0.5572 - val_accuracy: 0.8461 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5502 - accuracy: 0.8435 - val_loss: 0.5587 - val_accuracy: 0.8438 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5529 - accuracy: 0.8431 - val_loss: 0.5575 - val_accuracy: 0.8127 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5648 - accuracy: 0.8447 - val_loss: 0.5540 - val_accuracy: 0.8401 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5722 - accuracy: 0.8460 - val_loss: 0.5529 - val_accuracy: 0.8516 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5491 - accuracy: 0.8579 - val_loss: 0.5558 - val_accuracy: 0.8377 2938/2938 [==============================] - 2s 539us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7605 - accuracy: 0.4620 - val_loss: 0.6042 - val_accuracy: 0.7434 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5911 - accuracy: 0.7784 - val_loss: 0.5552 - val_accuracy: 0.8269 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5597 - accuracy: 0.8276 - val_loss: 0.5488 - val_accuracy: 0.8244 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5422 - accuracy: 0.8437 - val_loss: 0.5441 - val_accuracy: 0.8536 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5515 - accuracy: 0.8542 - val_loss: 0.5432 - val_accuracy: 0.8506 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5378 - accuracy: 0.8579 - val_loss: 0.5484 - val_accuracy: 0.8544 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5453 - accuracy: 0.8528 - val_loss: 0.5579 - val_accuracy: 0.7720 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5565 - accuracy: 0.8577 - val_loss: 0.5409 - val_accuracy: 0.8617 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5593 - accuracy: 0.8613 - val_loss: 0.5415 - val_accuracy: 0.8543 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5369 - accuracy: 0.8713 - val_loss: 0.5539 - val_accuracy: 0.8551 2938/2938 [==============================] - 2s 528us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7464 - accuracy: 0.5046 - val_loss: 0.5917 - val_accuracy: 0.8105 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.5858 - accuracy: 0.8345 - val_loss: 0.5750 - val_accuracy: 0.8530 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5848 - accuracy: 0.8525 - val_loss: 0.5613 - val_accuracy: 0.8445 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5506 - accuracy: 0.8778 - val_loss: 0.5550 - val_accuracy: 0.8728 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5554 - accuracy: 0.8851 - val_loss: 0.5535 - val_accuracy: 0.8840 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5441 - accuracy: 0.8976 - val_loss: 0.5571 - val_accuracy: 0.8486 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5452 - accuracy: 0.9023 - val_loss: 0.5499 - val_accuracy: 0.8807 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5627 - accuracy: 0.8978 - val_loss: 0.5471 - val_accuracy: 0.9020 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5803 - accuracy: 0.8885 - val_loss: 0.5443 - val_accuracy: 0.9204 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5407 - accuracy: 0.9108 - val_loss: 0.5633 - val_accuracy: 0.8704 2938/2938 [==============================] - 2s 539us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7518 - accuracy: 0.4986 - val_loss: 0.5950 - val_accuracy: 0.7868 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5900 - accuracy: 0.8051 - val_loss: 0.5843 - val_accuracy: 0.8154 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5828 - accuracy: 0.8278 - val_loss: 0.5761 - val_accuracy: 0.8248 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5680 - accuracy: 0.8343 - val_loss: 0.5720 - val_accuracy: 0.8532 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5742 - accuracy: 0.8476 - val_loss: 0.5702 - val_accuracy: 0.8501 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5634 - accuracy: 0.8577 - val_loss: 0.5713 - val_accuracy: 0.8436 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5655 - accuracy: 0.8608 - val_loss: 0.5717 - val_accuracy: 0.8214 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5778 - accuracy: 0.8533 - val_loss: 0.5591 - val_accuracy: 0.8732 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5795 - accuracy: 0.8585 - val_loss: 0.5658 - val_accuracy: 0.8512 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5602 - accuracy: 0.8594 - val_loss: 0.6071 - val_accuracy: 0.7387 2938/2938 [==============================] - 2s 546us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7448 - accuracy: 0.4881 - val_loss: 0.5892 - val_accuracy: 0.7705 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5845 - accuracy: 0.7938 - val_loss: 0.5703 - val_accuracy: 0.8245 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5757 - accuracy: 0.8273 - val_loss: 0.5682 - val_accuracy: 0.8396 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5590 - accuracy: 0.8400 - val_loss: 0.5771 - val_accuracy: 0.8099 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5684 - accuracy: 0.8430 - val_loss: 0.5636 - val_accuracy: 0.8493 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5569 - accuracy: 0.8556 - val_loss: 0.5677 - val_accuracy: 0.8376 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5589 - accuracy: 0.8624 - val_loss: 0.5625 - val_accuracy: 0.8418 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5694 - accuracy: 0.8720 - val_loss: 0.5579 - val_accuracy: 0.8520 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5824 - accuracy: 0.8569 - val_loss: 0.5586 - val_accuracy: 0.8722 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5542 - accuracy: 0.8844 - val_loss: 0.6013 - val_accuracy: 0.7820 2938/2938 [==============================] - 2s 549us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.7871 - accuracy: 0.4651 - val_loss: 0.6427 - val_accuracy: 0.7778 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6398 - accuracy: 0.7991 - val_loss: 0.6320 - val_accuracy: 0.8188 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.6415 - accuracy: 0.8267 - val_loss: 0.6286 - val_accuracy: 0.8243 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6200 - accuracy: 0.8414 - val_loss: 0.6249 - val_accuracy: 0.8452 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6398 - accuracy: 0.8240 - val_loss: 0.6345 - val_accuracy: 0.8304 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6273 - accuracy: 0.8286 - val_loss: 0.6342 - val_accuracy: 0.8258 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6310 - accuracy: 0.8325 - val_loss: 0.6384 - val_accuracy: 0.7925 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6583 - accuracy: 0.8197 - val_loss: 0.6237 - val_accuracy: 0.8490 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6660 - accuracy: 0.8094 - val_loss: 0.6233 - val_accuracy: 0.8555 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6223 - accuracy: 0.8538 - val_loss: 0.6497 - val_accuracy: 0.7521 2938/2938 [==============================] - 2s 553us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7682 - accuracy: 0.5059 - val_loss: 0.6147 - val_accuracy: 0.8106 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6062 - accuracy: 0.8525 - val_loss: 0.5867 - val_accuracy: 0.8936 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5918 - accuracy: 0.8832 - val_loss: 0.5907 - val_accuracy: 0.8569 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5802 - accuracy: 0.8891 - val_loss: 0.5830 - val_accuracy: 0.8897 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5947 - accuracy: 0.8844 - val_loss: 0.5821 - val_accuracy: 0.8957 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5792 - accuracy: 0.8938 - val_loss: 0.5812 - val_accuracy: 0.8930 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5823 - accuracy: 0.8926 - val_loss: 0.5928 - val_accuracy: 0.8642 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6041 - accuracy: 0.8821 - val_loss: 0.5823 - val_accuracy: 0.8988 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5974 - accuracy: 0.8877 - val_loss: 0.5801 - val_accuracy: 0.9005 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5841 - accuracy: 0.8872 - val_loss: 0.5875 - val_accuracy: 0.8484 2938/2938 [==============================] - 2s 533us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7609 - accuracy: 0.4881 - val_loss: 0.6121 - val_accuracy: 0.7783 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6063 - accuracy: 0.8021 - val_loss: 0.6108 - val_accuracy: 0.8282 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6082 - accuracy: 0.8265 - val_loss: 0.5950 - val_accuracy: 0.8263 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5905 - accuracy: 0.8361 - val_loss: 0.5867 - val_accuracy: 0.8493 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5907 - accuracy: 0.8441 - val_loss: 0.5861 - val_accuracy: 0.8408 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5812 - accuracy: 0.8491 - val_loss: 0.5841 - val_accuracy: 0.8580 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5816 - accuracy: 0.8563 - val_loss: 0.5862 - val_accuracy: 0.8304 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5999 - accuracy: 0.8502 - val_loss: 0.5833 - val_accuracy: 0.8543 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6102 - accuracy: 0.8419 - val_loss: 0.5798 - val_accuracy: 0.8623 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5758 - accuracy: 0.8590 - val_loss: 0.5916 - val_accuracy: 0.8038 2938/2938 [==============================] - 2s 520us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.7575 - accuracy: 0.4806 - val_loss: 0.6120 - val_accuracy: 0.7313 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.6058 - accuracy: 0.7513 - val_loss: 0.5911 - val_accuracy: 0.7741 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.5987 - accuracy: 0.7900 - val_loss: 0.5878 - val_accuracy: 0.7875 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5744 - accuracy: 0.8095 - val_loss: 0.5760 - val_accuracy: 0.8019 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5740 - accuracy: 0.8164 - val_loss: 0.5571 - val_accuracy: 0.8411 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5525 - accuracy: 0.8357 - val_loss: 0.5545 - val_accuracy: 0.8443 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5539 - accuracy: 0.8395 - val_loss: 0.5574 - val_accuracy: 0.8051 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5674 - accuracy: 0.8316 - val_loss: 0.5731 - val_accuracy: 0.8301 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5761 - accuracy: 0.8356 - val_loss: 0.5507 - val_accuracy: 0.8587 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.5479 - accuracy: 0.8506 - val_loss: 0.5563 - val_accuracy: 0.8265 2938/2938 [==============================] - 2s 541us/step
Mean average precision over 10 runs: 0.2306
[0.2293, 0.2492, 0.2382, 0.2484, 0.2316, 0.2273, 0.1948, 0.2115, 0.223, 0.2526]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.2306 Coefficient of variation over 10 runs: 0.0741
[0.2293, 0.2492, 0.2382, 0.2484, 0.2316, 0.2273, 0.1948, 0.2115, 0.223, 0.2526]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
# Apply linear activation function throughout
# Generate first hidden layer with 40 nodes
model.add(Dense(units=40, activation='linear',input_dim=29))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2050 - accuracy: 0.7910 - val_loss: 0.0011 - val_accuracy: 0.9762 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9896 - val_loss: 3.1096e-04 - val_accuracy: 0.9940 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0014 - accuracy: 0.9922 - val_loss: 0.0011 - val_accuracy: 0.9887 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.9722e-04 - accuracy: 0.9940 - val_loss: 1.0601e-06 - val_accuracy: 0.9995 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.7942e-04 - accuracy: 0.9939 - val_loss: 1.5084e-04 - val_accuracy: 0.9946 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.5940e-04 - accuracy: 0.9963 - val_loss: 0.0018 - val_accuracy: 0.9817 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4761e-04 - accuracy: 0.9940 - val_loss: 0.0036 - val_accuracy: 0.9795 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0037 - accuracy: 0.9882 - val_loss: 4.8492e-05 - val_accuracy: 0.9963 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9910 - val_loss: 0.0046 - val_accuracy: 0.9776 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.1011e-04 - accuracy: 0.9918 - val_loss: 0.0047 - val_accuracy: 0.9685 2938/2938 [==============================] - 2s 574us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2137 - accuracy: 0.7923 - val_loss: 0.0020 - val_accuracy: 0.9731 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9898 - val_loss: 2.1718e-04 - val_accuracy: 0.9936 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.9673e-04 - accuracy: 0.9927 - val_loss: 8.3525e-04 - val_accuracy: 0.9926 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.8769e-04 - accuracy: 0.9928 - val_loss: 2.6975e-06 - val_accuracy: 0.9993 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.5500e-04 - accuracy: 0.9933 - val_loss: 1.2976e-04 - val_accuracy: 0.9960 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 3.6985e-04 - accuracy: 0.9970 - val_loss: 0.0020 - val_accuracy: 0.9813 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.1426e-04 - accuracy: 0.9937 - val_loss: 0.0019 - val_accuracy: 0.9847 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9904 - val_loss: 1.5503e-05 - val_accuracy: 0.9981 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9926 - val_loss: 0.0059 - val_accuracy: 0.9780 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.2538e-04 - accuracy: 0.9918 - val_loss: 0.0035 - val_accuracy: 0.9702 2938/2938 [==============================] - 2s 509us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2071 - accuracy: 0.7888 - val_loss: 0.0024 - val_accuracy: 0.9706 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9902 - val_loss: 3.2068e-04 - val_accuracy: 0.9921 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.7611e-04 - accuracy: 0.9933 - val_loss: 7.4785e-04 - val_accuracy: 0.9914 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.2452e-04 - accuracy: 0.9942 - val_loss: 1.2649e-04 - val_accuracy: 0.9967 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 7.5666e-04 - accuracy: 0.9924 - val_loss: 1.0727e-04 - val_accuracy: 0.9958 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.9379e-04 - accuracy: 0.9971 - val_loss: 0.0013 - val_accuracy: 0.9829 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4980e-04 - accuracy: 0.9948 - val_loss: 0.0021 - val_accuracy: 0.9855 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9906 - val_loss: 2.3885e-05 - val_accuracy: 0.9976 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9921 - val_loss: 0.0018 - val_accuracy: 0.9888 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.7834e-04 - accuracy: 0.9928 - val_loss: 0.0044 - val_accuracy: 0.9710 2938/2938 [==============================] - 2s 500us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2383 - accuracy: 0.7792 - val_loss: 0.0107 - val_accuracy: 0.9478 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9897 - val_loss: 1.5511e-05 - val_accuracy: 0.9983 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.0681e-04 - accuracy: 0.9938 - val_loss: 8.4162e-04 - val_accuracy: 0.9918 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.6346e-04 - accuracy: 0.9944 - val_loss: 2.0078e-06 - val_accuracy: 0.9995 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.8650e-04 - accuracy: 0.9936 - val_loss: 1.3419e-04 - val_accuracy: 0.9960 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.2295e-04 - accuracy: 0.9971 - val_loss: 0.0011 - val_accuracy: 0.9862 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4715e-04 - accuracy: 0.9953 - val_loss: 0.0020 - val_accuracy: 0.9865 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9923 - val_loss: 1.9046e-05 - val_accuracy: 0.9985 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9937 - val_loss: 0.0014 - val_accuracy: 0.9905 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.4095e-04 - accuracy: 0.9930 - val_loss: 0.0011 - val_accuracy: 0.9819 2938/2938 [==============================] - 2s 500us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2179 - accuracy: 0.7812 - val_loss: 0.0039 - val_accuracy: 0.9601 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9895 - val_loss: 2.6589e-04 - val_accuracy: 0.9935 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9928 - val_loss: 0.0012 - val_accuracy: 0.9878 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.5432e-04 - accuracy: 0.9945 - val_loss: 5.0071e-06 - val_accuracy: 0.9991 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.0897e-04 - accuracy: 0.9938 - val_loss: 1.2628e-04 - val_accuracy: 0.9948 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.7272e-04 - accuracy: 0.9961 - val_loss: 5.3590e-04 - val_accuracy: 0.9887 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4928e-04 - accuracy: 0.9943 - val_loss: 0.0015 - val_accuracy: 0.9860 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9912 - val_loss: 2.7145e-05 - val_accuracy: 0.9974 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9928 - val_loss: 0.0027 - val_accuracy: 0.9856 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.9069e-04 - accuracy: 0.9919 - val_loss: 0.0023 - val_accuracy: 0.9764 2938/2938 [==============================] - 2s 505us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2295 - accuracy: 0.7788 - val_loss: 0.0011 - val_accuracy: 0.9789 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9900 - val_loss: 9.3789e-05 - val_accuracy: 0.9964 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.8081e-04 - accuracy: 0.9934 - val_loss: 0.0014 - val_accuracy: 0.9879 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.2028e-04 - accuracy: 0.9943 - val_loss: 3.9279e-06 - val_accuracy: 0.9992 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.5874e-04 - accuracy: 0.9937 - val_loss: 2.2210e-04 - val_accuracy: 0.9937 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.1237e-04 - accuracy: 0.9959 - val_loss: 9.2732e-04 - val_accuracy: 0.9873 Epoch 7/10 5964/5964 [==============================] - 11s 2ms/step - loss: 4.1026e-04 - accuracy: 0.9952 - val_loss: 0.0053 - val_accuracy: 0.9724 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0038 - accuracy: 0.9880 - val_loss: 1.8961e-05 - val_accuracy: 0.9982 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9931 - val_loss: 0.0036 - val_accuracy: 0.9844 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.0874e-04 - accuracy: 0.9923 - val_loss: 0.0023 - val_accuracy: 0.9795 2938/2938 [==============================] - 2s 563us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2205 - accuracy: 0.7721 - val_loss: 1.8490e-04 - val_accuracy: 0.9909 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9910 - val_loss: 0.0022 - val_accuracy: 0.9823 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9919 - val_loss: 0.0019 - val_accuracy: 0.9860 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.8025e-04 - accuracy: 0.9949 - val_loss: 3.5713e-06 - val_accuracy: 0.9993 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.5930e-04 - accuracy: 0.9932 - val_loss: 5.2104e-05 - val_accuracy: 0.9968 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.3828e-04 - accuracy: 0.9962 - val_loss: 0.0011 - val_accuracy: 0.9837 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.4112e-04 - accuracy: 0.9939 - val_loss: 0.0010 - val_accuracy: 0.9900 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9915 - val_loss: 2.7616e-05 - val_accuracy: 0.9978 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9920 - val_loss: 0.0024 - val_accuracy: 0.9811 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.9179e-04 - accuracy: 0.9921 - val_loss: 0.0022 - val_accuracy: 0.9783 2938/2938 [==============================] - 2s 508us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2146 - accuracy: 0.8073 - val_loss: 0.0041 - val_accuracy: 0.9638 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9911 - val_loss: 2.0278e-04 - val_accuracy: 0.9943 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0011 - accuracy: 0.9922 - val_loss: 8.4932e-04 - val_accuracy: 0.9894 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 5.8817e-04 - accuracy: 0.9940 - val_loss: 8.1787e-06 - val_accuracy: 0.9988 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.4110e-04 - accuracy: 0.9945 - val_loss: 5.9755e-05 - val_accuracy: 0.9966 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.5209e-04 - accuracy: 0.9960 - val_loss: 9.7591e-04 - val_accuracy: 0.9875 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.7731e-04 - accuracy: 0.9951 - val_loss: 0.0025 - val_accuracy: 0.9853 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9901 - val_loss: 1.2515e-05 - val_accuracy: 0.9988 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9940 - val_loss: 0.0038 - val_accuracy: 0.9830 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 6.9216e-04 - accuracy: 0.9927 - val_loss: 0.0051 - val_accuracy: 0.9692 2938/2938 [==============================] - 2s 506us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2006 - accuracy: 0.7866 - val_loss: 0.0018 - val_accuracy: 0.9726 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9903 - val_loss: 3.6475e-05 - val_accuracy: 0.9974 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 8.0648e-04 - accuracy: 0.9929 - val_loss: 3.2045e-04 - val_accuracy: 0.9940 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 4.5670e-04 - accuracy: 0.9945 - val_loss: 3.0967e-06 - val_accuracy: 0.9991 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.3938e-04 - accuracy: 0.9937 - val_loss: 1.0909e-04 - val_accuracy: 0.9950 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.3737e-04 - accuracy: 0.9959 - val_loss: 6.8811e-04 - val_accuracy: 0.9883 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.4069e-04 - accuracy: 0.9955 - val_loss: 0.0022 - val_accuracy: 0.9847 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0034 - accuracy: 0.9900 - val_loss: 2.1078e-04 - val_accuracy: 0.9942 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9938 - val_loss: 0.0011 - val_accuracy: 0.9897 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 6.8268e-04 - accuracy: 0.9926 - val_loss: 0.0031 - val_accuracy: 0.9738 2938/2938 [==============================] - 1s 489us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2184 - accuracy: 0.7877 - val_loss: 0.0075 - val_accuracy: 0.9587 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9912 - val_loss: 6.5324e-05 - val_accuracy: 0.9963 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 9.6679e-04 - accuracy: 0.9928 - val_loss: 5.4989e-04 - val_accuracy: 0.9914 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.0135e-04 - accuracy: 0.9929 - val_loss: 7.6205e-06 - val_accuracy: 0.9990 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 5.8127e-04 - accuracy: 0.9929 - val_loss: 5.5006e-05 - val_accuracy: 0.9960 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.5529e-04 - accuracy: 0.9960 - val_loss: 8.9819e-04 - val_accuracy: 0.9855 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 3.9595e-04 - accuracy: 0.9956 - val_loss: 0.0032 - val_accuracy: 0.9821 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9902 - val_loss: 1.4756e-04 - val_accuracy: 0.9956 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9925 - val_loss: 0.0021 - val_accuracy: 0.9845 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 7.3236e-04 - accuracy: 0.9922 - val_loss: 0.0027 - val_accuracy: 0.9694 2938/2938 [==============================] - 1s 484us/step
Mean average precision over 10 runs: 0.0362
[0.0226, 0.0259, 0.0469, 0.0233, 0.0467, 0.0167, 0.0349, 0.0419, 0.0373, 0.0658]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0362 Coefficient of variation over 10 runs: 0.3885
[0.0226, 0.0259, 0.0469, 0.0233, 0.0467, 0.0167, 0.0349, 0.0419, 0.0373, 0.0658]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
model.add(Dense(units=40, activation='linear',input_dim=29))
model.add(Dropout(0.10))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2984 - accuracy: 0.6287 - val_loss: 0.0101 - val_accuracy: 0.9635 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7832 - val_loss: 0.0077 - val_accuracy: 0.9731 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7832 - val_loss: 0.0091 - val_accuracy: 0.9727 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.7827 - val_loss: 0.0091 - val_accuracy: 0.9688 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7835 - val_loss: 0.0075 - val_accuracy: 0.9739 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0760 - accuracy: 0.7846 - val_loss: 0.0074 - val_accuracy: 0.9721 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7854 - val_loss: 0.0085 - val_accuracy: 0.9694 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0895 - accuracy: 0.7837 - val_loss: 0.0098 - val_accuracy: 0.9607 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0787 - accuracy: 0.7856 - val_loss: 0.0090 - val_accuracy: 0.9687 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7847 - val_loss: 0.0123 - val_accuracy: 0.9678 2938/2938 [==============================] - 2s 504us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3124 - accuracy: 0.6303 - val_loss: 0.0098 - val_accuracy: 0.9694 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0767 - accuracy: 0.7856 - val_loss: 0.0083 - val_accuracy: 0.9704 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7865 - val_loss: 0.0093 - val_accuracy: 0.9719 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0761 - accuracy: 0.7862 - val_loss: 0.0103 - val_accuracy: 0.9639 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0768 - accuracy: 0.7868 - val_loss: 0.0075 - val_accuracy: 0.9728 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0751 - accuracy: 0.7883 - val_loss: 0.0093 - val_accuracy: 0.9678 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7860 - val_loss: 0.0096 - val_accuracy: 0.9679 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0868 - accuracy: 0.7868 - val_loss: 0.0106 - val_accuracy: 0.9634 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0870 - accuracy: 0.7847 - val_loss: 0.0082 - val_accuracy: 0.9749 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0761 - accuracy: 0.7866 - val_loss: 0.0146 - val_accuracy: 0.9615 2938/2938 [==============================] - 1s 483us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3070 - accuracy: 0.6297 - val_loss: 0.0093 - val_accuracy: 0.9671 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0766 - accuracy: 0.7882 - val_loss: 0.0080 - val_accuracy: 0.9702 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0769 - accuracy: 0.7887 - val_loss: 0.0092 - val_accuracy: 0.9733 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0743 - accuracy: 0.7868 - val_loss: 0.0099 - val_accuracy: 0.9647 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0775 - accuracy: 0.7862 - val_loss: 0.0079 - val_accuracy: 0.9742 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0745 - accuracy: 0.7896 - val_loss: 0.0089 - val_accuracy: 0.9706 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0778 - accuracy: 0.7865 - val_loss: 0.0084 - val_accuracy: 0.9685 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0809 - accuracy: 0.7837 - val_loss: 0.0091 - val_accuracy: 0.9668 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0911 - accuracy: 0.7836 - val_loss: 0.0089 - val_accuracy: 0.9685 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0756 - accuracy: 0.7843 - val_loss: 0.0136 - val_accuracy: 0.9624 2938/2938 [==============================] - 1s 486us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3393 - accuracy: 0.6201 - val_loss: 0.0091 - val_accuracy: 0.9690 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0752 - accuracy: 0.7860 - val_loss: 0.0081 - val_accuracy: 0.9725 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0759 - accuracy: 0.7847 - val_loss: 0.0082 - val_accuracy: 0.9740 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.7863 - val_loss: 0.0089 - val_accuracy: 0.9665 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0798 - accuracy: 0.7827 - val_loss: 0.0071 - val_accuracy: 0.9762 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0746 - accuracy: 0.7865 - val_loss: 0.0084 - val_accuracy: 0.9677 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0763 - accuracy: 0.7827 - val_loss: 0.0073 - val_accuracy: 0.9783 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0866 - accuracy: 0.7852 - val_loss: 0.0109 - val_accuracy: 0.9575 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0813 - accuracy: 0.7844 - val_loss: 0.0088 - val_accuracy: 0.9686 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0746 - accuracy: 0.7849 - val_loss: 0.0117 - val_accuracy: 0.9654 2938/2938 [==============================] - 1s 484us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3210 - accuracy: 0.6188 - val_loss: 0.0093 - val_accuracy: 0.9704 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0763 - accuracy: 0.7866 - val_loss: 0.0078 - val_accuracy: 0.9743 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7863 - val_loss: 0.0083 - val_accuracy: 0.9732 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0733 - accuracy: 0.7865 - val_loss: 0.0113 - val_accuracy: 0.9624 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0785 - accuracy: 0.7826 - val_loss: 0.0076 - val_accuracy: 0.9757 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0749 - accuracy: 0.7866 - val_loss: 0.0089 - val_accuracy: 0.9698 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0767 - accuracy: 0.7852 - val_loss: 0.0075 - val_accuracy: 0.9744 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0962 - accuracy: 0.7829 - val_loss: 0.0084 - val_accuracy: 0.9720 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0819 - accuracy: 0.7833 - val_loss: 0.0089 - val_accuracy: 0.9686 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0753 - accuracy: 0.7848 - val_loss: 0.0160 - val_accuracy: 0.9613 2938/2938 [==============================] - 1s 487us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3324 - accuracy: 0.6174 - val_loss: 0.0096 - val_accuracy: 0.9675 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0772 - accuracy: 0.7850 - val_loss: 0.0075 - val_accuracy: 0.9751 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0770 - accuracy: 0.7866 - val_loss: 0.0092 - val_accuracy: 0.9710 Epoch 4/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0745 - accuracy: 0.7875 - val_loss: 0.0093 - val_accuracy: 0.9684 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7855 - val_loss: 0.0071 - val_accuracy: 0.9755 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0743 - accuracy: 0.7873 - val_loss: 0.0096 - val_accuracy: 0.9660 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7836 - val_loss: 0.0081 - val_accuracy: 0.9723 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.1045 - accuracy: 0.7827 - val_loss: 0.0092 - val_accuracy: 0.9638 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0876 - accuracy: 0.7870 - val_loss: 0.0095 - val_accuracy: 0.9666 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7867 - val_loss: 0.0127 - val_accuracy: 0.9662 2938/2938 [==============================] - 1s 487us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3154 - accuracy: 0.6137 - val_loss: 0.0088 - val_accuracy: 0.9717 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7849 - val_loss: 0.0077 - val_accuracy: 0.9716 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0763 - accuracy: 0.7868 - val_loss: 0.0088 - val_accuracy: 0.9733 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0751 - accuracy: 0.7884 - val_loss: 0.0107 - val_accuracy: 0.9672 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7850 - val_loss: 0.0076 - val_accuracy: 0.9748 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0759 - accuracy: 0.7845 - val_loss: 0.0077 - val_accuracy: 0.9739 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0758 - accuracy: 0.7842 - val_loss: 0.0085 - val_accuracy: 0.9730 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0920 - accuracy: 0.7840 - val_loss: 0.0103 - val_accuracy: 0.9546 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0855 - accuracy: 0.7843 - val_loss: 0.0080 - val_accuracy: 0.9722 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0769 - accuracy: 0.7893 - val_loss: 0.0123 - val_accuracy: 0.9618 2938/2938 [==============================] - 1s 495us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3157 - accuracy: 0.6396 - val_loss: 0.0091 - val_accuracy: 0.9680 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0765 - accuracy: 0.7870 - val_loss: 0.0077 - val_accuracy: 0.9744 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7884 - val_loss: 0.0082 - val_accuracy: 0.9723 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0741 - accuracy: 0.7852 - val_loss: 0.0094 - val_accuracy: 0.9703 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0786 - accuracy: 0.7860 - val_loss: 0.0075 - val_accuracy: 0.9771 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0753 - accuracy: 0.7870 - val_loss: 0.0076 - val_accuracy: 0.9719 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0764 - accuracy: 0.7878 - val_loss: 0.0090 - val_accuracy: 0.9672 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0851 - accuracy: 0.7828 - val_loss: 0.0094 - val_accuracy: 0.9661 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0789 - accuracy: 0.7858 - val_loss: 0.0100 - val_accuracy: 0.9690 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0745 - accuracy: 0.7872 - val_loss: 0.0121 - val_accuracy: 0.9619 2938/2938 [==============================] - 1s 493us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3057 - accuracy: 0.6235 - val_loss: 0.0088 - val_accuracy: 0.9673 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7856 - val_loss: 0.0079 - val_accuracy: 0.9723 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0766 - accuracy: 0.7863 - val_loss: 0.0087 - val_accuracy: 0.9730 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0732 - accuracy: 0.7872 - val_loss: 0.0090 - val_accuracy: 0.9690 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0770 - accuracy: 0.7847 - val_loss: 0.0074 - val_accuracy: 0.9727 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0762 - accuracy: 0.7843 - val_loss: 0.0080 - val_accuracy: 0.9711 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0775 - accuracy: 0.7832 - val_loss: 0.0080 - val_accuracy: 0.9732 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0970 - accuracy: 0.7838 - val_loss: 0.0097 - val_accuracy: 0.9678 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0821 - accuracy: 0.7863 - val_loss: 0.0094 - val_accuracy: 0.9706 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0752 - accuracy: 0.7837 - val_loss: 0.0141 - val_accuracy: 0.9587 2938/2938 [==============================] - 2s 510us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3187 - accuracy: 0.6286 - val_loss: 0.0092 - val_accuracy: 0.9706 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7853 - val_loss: 0.0078 - val_accuracy: 0.9685 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0772 - accuracy: 0.7864 - val_loss: 0.0082 - val_accuracy: 0.9750 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.7875 - val_loss: 0.0095 - val_accuracy: 0.9682 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0787 - accuracy: 0.7878 - val_loss: 0.0072 - val_accuracy: 0.9772 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7864 - val_loss: 0.0080 - val_accuracy: 0.9723 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0769 - accuracy: 0.7856 - val_loss: 0.0076 - val_accuracy: 0.9749 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0952 - accuracy: 0.7847 - val_loss: 0.0090 - val_accuracy: 0.9670 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0842 - accuracy: 0.7824 - val_loss: 0.0083 - val_accuracy: 0.9736 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0755 - accuracy: 0.7837 - val_loss: 0.0139 - val_accuracy: 0.9571 2938/2938 [==============================] - 2s 580us/step
Mean average precision over 10 runs: 0.1372
[0.1539, 0.1048, 0.0913, 0.1385, 0.1235, 0.1244, 0.2411, 0.1668, 0.1482, 0.0798]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.1372 Coefficient of variation over 10 runs: 0.3167
[0.1539, 0.1048, 0.0913, 0.1385, 0.1235, 0.1244, 0.2411, 0.1668, 0.1482, 0.0798]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
model.add(Dense(units=40, activation='linear', \
activity_regularizer=regularizers.l1(10e-5), input_dim=29))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2077 - accuracy: 0.7909 - val_loss: 0.0038 - val_accuracy: 0.9781 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0041 - accuracy: 0.9898 - val_loss: 0.0023 - val_accuracy: 0.9947 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0031 - accuracy: 0.9931 - val_loss: 0.0037 - val_accuracy: 0.9851 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9932 - val_loss: 0.0020 - val_accuracy: 0.9972 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9913 - val_loss: 0.0017 - val_accuracy: 0.9947 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9950 - val_loss: 0.0032 - val_accuracy: 0.9827 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9934 - val_loss: 0.0061 - val_accuracy: 0.9747 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0054 - accuracy: 0.9859 - val_loss: 0.0015 - val_accuracy: 0.9939 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9911 - val_loss: 0.0069 - val_accuracy: 0.9757 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9900 - val_loss: 0.0074 - val_accuracy: 0.9643 2938/2938 [==============================] - 1s 491us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2164 - accuracy: 0.7922 - val_loss: 0.0054 - val_accuracy: 0.9695 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0044 - accuracy: 0.9898 - val_loss: 0.0022 - val_accuracy: 0.9964 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9938 - val_loss: 0.0025 - val_accuracy: 0.9924 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9930 - val_loss: 0.0019 - val_accuracy: 0.9968 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9912 - val_loss: 0.0016 - val_accuracy: 0.9951 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9954 - val_loss: 0.0031 - val_accuracy: 0.9797 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9929 - val_loss: 0.0036 - val_accuracy: 0.9818 Epoch 8/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0046 - accuracy: 0.9881 - val_loss: 0.0014 - val_accuracy: 0.9981 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0041 - accuracy: 0.9914 - val_loss: 0.0079 - val_accuracy: 0.9769 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0025 - accuracy: 0.9895 - val_loss: 0.0076 - val_accuracy: 0.9602 2938/2938 [==============================] - 1s 494us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2097 - accuracy: 0.7888 - val_loss: 0.0045 - val_accuracy: 0.9725 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9907 - val_loss: 0.0024 - val_accuracy: 0.9921 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9925 - val_loss: 0.0027 - val_accuracy: 0.9889 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9939 - val_loss: 0.0017 - val_accuracy: 0.9989 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9929 - val_loss: 0.0016 - val_accuracy: 0.9967 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9953 - val_loss: 0.0035 - val_accuracy: 0.9798 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9927 - val_loss: 0.0049 - val_accuracy: 0.9792 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0049 - accuracy: 0.9876 - val_loss: 0.0013 - val_accuracy: 0.9984 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0034 - accuracy: 0.9915 - val_loss: 0.0067 - val_accuracy: 0.9771 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9896 - val_loss: 0.0040 - val_accuracy: 0.9731 2938/2938 [==============================] - 2s 495us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2410 - accuracy: 0.7792 - val_loss: 0.0138 - val_accuracy: 0.9481 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9897 - val_loss: 0.0022 - val_accuracy: 0.9960 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9927 - val_loss: 0.0021 - val_accuracy: 0.9950 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9947 - val_loss: 0.0022 - val_accuracy: 0.9946 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9903 - val_loss: 0.0016 - val_accuracy: 0.9964 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9951 - val_loss: 0.0034 - val_accuracy: 0.9780 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9929 - val_loss: 0.0041 - val_accuracy: 0.9797 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9877 - val_loss: 0.0013 - val_accuracy: 0.9980 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0035 - accuracy: 0.9907 - val_loss: 0.0075 - val_accuracy: 0.9703 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9888 - val_loss: 0.0055 - val_accuracy: 0.9705 2938/2938 [==============================] - 1s 485us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2206 - accuracy: 0.7812 - val_loss: 0.0058 - val_accuracy: 0.9632 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9902 - val_loss: 0.0022 - val_accuracy: 0.9964 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9925 - val_loss: 0.0038 - val_accuracy: 0.9843 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9928 - val_loss: 0.0016 - val_accuracy: 0.9989 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9913 - val_loss: 0.0016 - val_accuracy: 0.9951 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0019 - accuracy: 0.9950 - val_loss: 0.0030 - val_accuracy: 0.9800 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0021 - accuracy: 0.9930 - val_loss: 0.0029 - val_accuracy: 0.9856 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9874 - val_loss: 0.0014 - val_accuracy: 0.9960 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0045 - accuracy: 0.9911 - val_loss: 0.0066 - val_accuracy: 0.9775 Epoch 10/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0025 - accuracy: 0.9895 - val_loss: 0.0060 - val_accuracy: 0.9633 2938/2938 [==============================] - 1s 490us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2321 - accuracy: 0.7788 - val_loss: 0.0039 - val_accuracy: 0.9768 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9895 - val_loss: 0.0022 - val_accuracy: 0.9947 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0030 - accuracy: 0.9924 - val_loss: 0.0027 - val_accuracy: 0.9874 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9937 - val_loss: 0.0017 - val_accuracy: 0.9990 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9925 - val_loss: 0.0016 - val_accuracy: 0.9953 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9953 - val_loss: 0.0038 - val_accuracy: 0.9755 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9931 - val_loss: 0.0040 - val_accuracy: 0.9814 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9882 - val_loss: 0.0015 - val_accuracy: 0.9940 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0047 - accuracy: 0.9905 - val_loss: 0.0092 - val_accuracy: 0.9716 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9892 - val_loss: 0.0053 - val_accuracy: 0.9669 2938/2938 [==============================] - 1s 495us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2231 - accuracy: 0.7721 - val_loss: 0.0028 - val_accuracy: 0.9888 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0042 - accuracy: 0.9906 - val_loss: 0.0023 - val_accuracy: 0.9944 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9928 - val_loss: 0.0038 - val_accuracy: 0.9847 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9942 - val_loss: 0.0021 - val_accuracy: 0.9960 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9910 - val_loss: 0.0016 - val_accuracy: 0.9943 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9950 - val_loss: 0.0027 - val_accuracy: 0.9824 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9926 - val_loss: 0.0032 - val_accuracy: 0.9867 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0049 - accuracy: 0.9872 - val_loss: 0.0014 - val_accuracy: 0.9963 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9907 - val_loss: 0.0098 - val_accuracy: 0.9699 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9891 - val_loss: 0.0069 - val_accuracy: 0.9637 2938/2938 [==============================] - 1s 489us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2173 - accuracy: 0.8072 - val_loss: 0.0041 - val_accuracy: 0.9791 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9908 - val_loss: 0.0024 - val_accuracy: 0.9936 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0031 - accuracy: 0.9926 - val_loss: 0.0029 - val_accuracy: 0.9884 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9934 - val_loss: 0.0017 - val_accuracy: 0.9986 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9924 - val_loss: 0.0016 - val_accuracy: 0.9950 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9952 - val_loss: 0.0033 - val_accuracy: 0.9814 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9939 - val_loss: 0.0036 - val_accuracy: 0.9829 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9874 - val_loss: 0.0016 - val_accuracy: 0.9930 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0043 - accuracy: 0.9897 - val_loss: 0.0088 - val_accuracy: 0.9729 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9900 - val_loss: 0.0052 - val_accuracy: 0.9720 2938/2938 [==============================] - 1s 488us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2034 - accuracy: 0.7866 - val_loss: 0.0040 - val_accuracy: 0.9757 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9911 - val_loss: 0.0022 - val_accuracy: 0.9968 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0031 - accuracy: 0.9929 - val_loss: 0.0027 - val_accuracy: 0.9900 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9937 - val_loss: 0.0017 - val_accuracy: 0.9990 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9925 - val_loss: 0.0016 - val_accuracy: 0.9962 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9948 - val_loss: 0.0035 - val_accuracy: 0.9785 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9923 - val_loss: 0.0037 - val_accuracy: 0.9825 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0047 - accuracy: 0.9871 - val_loss: 0.0014 - val_accuracy: 0.9972 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9904 - val_loss: 0.0072 - val_accuracy: 0.9749 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9893 - val_loss: 0.0066 - val_accuracy: 0.9661 2938/2938 [==============================] - 1s 484us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2211 - accuracy: 0.7877 - val_loss: 0.0096 - val_accuracy: 0.9624 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0036 - accuracy: 0.9915 - val_loss: 0.0023 - val_accuracy: 0.9935 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0030 - accuracy: 0.9931 - val_loss: 0.0029 - val_accuracy: 0.9871 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9940 - val_loss: 0.0017 - val_accuracy: 0.9990 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9923 - val_loss: 0.0016 - val_accuracy: 0.9949 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9953 - val_loss: 0.0030 - val_accuracy: 0.9775 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9932 - val_loss: 0.0048 - val_accuracy: 0.9797 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9878 - val_loss: 0.0014 - val_accuracy: 0.9970 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9906 - val_loss: 0.0070 - val_accuracy: 0.9744 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0025 - accuracy: 0.9890 - val_loss: 0.0031 - val_accuracy: 0.9739 2938/2938 [==============================] - 1s 494us/step
Mean average precision over 10 runs: 0.0334
[0.0227, 0.0438, 0.0229, 0.0462, 0.0307, 0.0293, 0.042, 0.0296, 0.0367, 0.0303]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0334 Coefficient of variation over 10 runs: 0.2375
[0.0227, 0.0438, 0.0229, 0.0462, 0.0307, 0.0293, 0.042, 0.0296, 0.0367, 0.0303]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
# Call neural network API
model = Sequential()
model.add(Dense(units=40, activation='linear', \
activity_regularizer=regularizers.l1(10e-5), input_dim=29))
model.add(Dropout(0.05))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE, y=X_train_AE,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2588 - accuracy: 0.6891 - val_loss: 0.0079 - val_accuracy: 0.9656 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.8589 - val_loss: 0.0051 - val_accuracy: 0.9793 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8616 - val_loss: 0.0061 - val_accuracy: 0.9774 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8590 - val_loss: 0.0058 - val_accuracy: 0.9779 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0405 - accuracy: 0.8593 - val_loss: 0.0043 - val_accuracy: 0.9799 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0399 - accuracy: 0.8601 - val_loss: 0.0052 - val_accuracy: 0.9767 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8627 - val_loss: 0.0050 - val_accuracy: 0.9758 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0450 - accuracy: 0.8597 - val_loss: 0.0074 - val_accuracy: 0.9649 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0428 - accuracy: 0.8590 - val_loss: 0.0053 - val_accuracy: 0.9781 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0407 - accuracy: 0.8629 - val_loss: 0.0123 - val_accuracy: 0.9602 2938/2938 [==============================] - 1s 485us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2650 - accuracy: 0.6930 - val_loss: 0.0080 - val_accuracy: 0.9709 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0406 - accuracy: 0.8611 - val_loss: 0.0061 - val_accuracy: 0.9735 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0404 - accuracy: 0.8626 - val_loss: 0.0066 - val_accuracy: 0.9779 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.8616 - val_loss: 0.0069 - val_accuracy: 0.9719 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0410 - accuracy: 0.8605 - val_loss: 0.0043 - val_accuracy: 0.9830 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8632 - val_loss: 0.0060 - val_accuracy: 0.9714 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8628 - val_loss: 0.0105 - val_accuracy: 0.9612 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0460 - accuracy: 0.8603 - val_loss: 0.0099 - val_accuracy: 0.9652 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0529 - accuracy: 0.8602 - val_loss: 0.0050 - val_accuracy: 0.9816 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.8614 - val_loss: 0.0089 - val_accuracy: 0.9718 2938/2938 [==============================] - 2s 508us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2607 - accuracy: 0.6899 - val_loss: 0.0070 - val_accuracy: 0.9721 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0409 - accuracy: 0.8624 - val_loss: 0.0050 - val_accuracy: 0.9787 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0411 - accuracy: 0.8630 - val_loss: 0.0056 - val_accuracy: 0.9804 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0394 - accuracy: 0.8636 - val_loss: 0.0079 - val_accuracy: 0.9681 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0410 - accuracy: 0.8626 - val_loss: 0.0044 - val_accuracy: 0.9811 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8650 - val_loss: 0.0055 - val_accuracy: 0.9747 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8627 - val_loss: 0.0056 - val_accuracy: 0.9761 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0436 - accuracy: 0.8599 - val_loss: 0.0067 - val_accuracy: 0.9693 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0507 - accuracy: 0.8610 - val_loss: 0.0057 - val_accuracy: 0.9758 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0384 - accuracy: 0.8628 - val_loss: 0.0057 - val_accuracy: 0.9781 2938/2938 [==============================] - 2s 524us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2940 - accuracy: 0.6803 - val_loss: 0.0072 - val_accuracy: 0.9722 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.8616 - val_loss: 0.0058 - val_accuracy: 0.9759 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8617 - val_loss: 0.0046 - val_accuracy: 0.9824 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0403 - accuracy: 0.8620 - val_loss: 0.0066 - val_accuracy: 0.9695 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0414 - accuracy: 0.8598 - val_loss: 0.0044 - val_accuracy: 0.9825 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8613 - val_loss: 0.0062 - val_accuracy: 0.9695 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0401 - accuracy: 0.8618 - val_loss: 0.0045 - val_accuracy: 0.9772 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0444 - accuracy: 0.8628 - val_loss: 0.0076 - val_accuracy: 0.9633 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8602 - val_loss: 0.0048 - val_accuracy: 0.9804 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8609 - val_loss: 0.0114 - val_accuracy: 0.9661 2938/2938 [==============================] - 1s 486us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2734 - accuracy: 0.6799 - val_loss: 0.0069 - val_accuracy: 0.9730 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.8605 - val_loss: 0.0047 - val_accuracy: 0.9819 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.8602 - val_loss: 0.0061 - val_accuracy: 0.9763 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.8619 - val_loss: 0.0091 - val_accuracy: 0.9727 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8614 - val_loss: 0.0053 - val_accuracy: 0.9757 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0388 - accuracy: 0.8645 - val_loss: 0.0049 - val_accuracy: 0.9802 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0396 - accuracy: 0.8618 - val_loss: 0.0069 - val_accuracy: 0.9704 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0522 - accuracy: 0.8593 - val_loss: 0.0061 - val_accuracy: 0.9718 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8611 - val_loss: 0.0066 - val_accuracy: 0.9750 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8632 - val_loss: 0.0088 - val_accuracy: 0.9721 2938/2938 [==============================] - 1s 493us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2847 - accuracy: 0.6774 - val_loss: 0.0079 - val_accuracy: 0.9692 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0421 - accuracy: 0.8606 - val_loss: 0.0047 - val_accuracy: 0.9814 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8628 - val_loss: 0.0052 - val_accuracy: 0.9784 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8626 - val_loss: 0.0061 - val_accuracy: 0.9715 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8619 - val_loss: 0.0041 - val_accuracy: 0.9845 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.8624 - val_loss: 0.0073 - val_accuracy: 0.9717 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8627 - val_loss: 0.0047 - val_accuracy: 0.9811 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0570 - accuracy: 0.8607 - val_loss: 0.0065 - val_accuracy: 0.9683 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0453 - accuracy: 0.8624 - val_loss: 0.0047 - val_accuracy: 0.9804 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8623 - val_loss: 0.0084 - val_accuracy: 0.9695 2938/2938 [==============================] - 1s 484us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2723 - accuracy: 0.6726 - val_loss: 0.0067 - val_accuracy: 0.9733 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0417 - accuracy: 0.8606 - val_loss: 0.0048 - val_accuracy: 0.9803 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.8627 - val_loss: 0.0052 - val_accuracy: 0.9783 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0392 - accuracy: 0.8631 - val_loss: 0.0080 - val_accuracy: 0.9717 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0399 - accuracy: 0.8623 - val_loss: 0.0046 - val_accuracy: 0.9789 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8624 - val_loss: 0.0056 - val_accuracy: 0.9738 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.8603 - val_loss: 0.0053 - val_accuracy: 0.9757 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0473 - accuracy: 0.8601 - val_loss: 0.0059 - val_accuracy: 0.9698 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0446 - accuracy: 0.8592 - val_loss: 0.0050 - val_accuracy: 0.9773 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8631 - val_loss: 0.0073 - val_accuracy: 0.9766 2938/2938 [==============================] - 2s 508us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2695 - accuracy: 0.7016 - val_loss: 0.0068 - val_accuracy: 0.9726 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8629 - val_loss: 0.0051 - val_accuracy: 0.9790 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8616 - val_loss: 0.0051 - val_accuracy: 0.9794 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8621 - val_loss: 0.0068 - val_accuracy: 0.9743 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0420 - accuracy: 0.8609 - val_loss: 0.0041 - val_accuracy: 0.9842 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0392 - accuracy: 0.8639 - val_loss: 0.0055 - val_accuracy: 0.9759 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.8646 - val_loss: 0.0059 - val_accuracy: 0.9740 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0482 - accuracy: 0.8617 - val_loss: 0.0068 - val_accuracy: 0.9698 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0438 - accuracy: 0.8617 - val_loss: 0.0063 - val_accuracy: 0.9749 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0393 - accuracy: 0.8631 - val_loss: 0.0079 - val_accuracy: 0.9765 2938/2938 [==============================] - 1s 488us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2599 - accuracy: 0.6842 - val_loss: 0.0072 - val_accuracy: 0.9710 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0419 - accuracy: 0.8613 - val_loss: 0.0052 - val_accuracy: 0.9771 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8626 - val_loss: 0.0054 - val_accuracy: 0.9798 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8613 - val_loss: 0.0052 - val_accuracy: 0.9772 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8592 - val_loss: 0.0044 - val_accuracy: 0.9796 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8611 - val_loss: 0.0050 - val_accuracy: 0.9770 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8612 - val_loss: 0.0058 - val_accuracy: 0.9719 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0526 - accuracy: 0.8602 - val_loss: 0.0064 - val_accuracy: 0.9706 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0418 - accuracy: 0.8615 - val_loss: 0.0053 - val_accuracy: 0.9753 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.8609 - val_loss: 0.0116 - val_accuracy: 0.9683 2938/2938 [==============================] - 2s 498us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2704 - accuracy: 0.6907 - val_loss: 0.0071 - val_accuracy: 0.9742 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0416 - accuracy: 0.8629 - val_loss: 0.0044 - val_accuracy: 0.9821 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0408 - accuracy: 0.8626 - val_loss: 0.0046 - val_accuracy: 0.9819 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.8639 - val_loss: 0.0078 - val_accuracy: 0.9720 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8642 - val_loss: 0.0041 - val_accuracy: 0.9826 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.8634 - val_loss: 0.0051 - val_accuracy: 0.9781 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8647 - val_loss: 0.0056 - val_accuracy: 0.9753 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0463 - accuracy: 0.8605 - val_loss: 0.0073 - val_accuracy: 0.9672 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0461 - accuracy: 0.8630 - val_loss: 0.0049 - val_accuracy: 0.9798 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.8617 - val_loss: 0.0132 - val_accuracy: 0.9680 2938/2938 [==============================] - 2s 547us/step
Mean average precision over 10 runs: 0.0798
[0.0522, 0.0791, 0.0592, 0.0826, 0.0863, 0.0864, 0.076, 0.1203, 0.073, 0.0831]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0798 Coefficient of variation over 10 runs: 0.2169
[0.0522, 0.0791, 0.0592, 0.0826, 0.0863, 0.0864, 0.076, 0.1203, 0.073, 0.0831]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape)
for i in range(0,10):
# Call neural network API
model = Sequential()
# Generate hidden layer with 27 nodes using linear activation
model.add(Dense(units=27, activation='linear', input_dim=29))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test_AE_noisy, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3728 - accuracy: 0.6286 - val_loss: 0.0365 - val_accuracy: 0.8961 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.8654 - val_loss: 0.0362 - val_accuracy: 0.9092 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0575 - accuracy: 0.8698 - val_loss: 0.0356 - val_accuracy: 0.8923 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.8712 - val_loss: 0.0365 - val_accuracy: 0.9097 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.8823 - val_loss: 0.0362 - val_accuracy: 0.9192 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0544 - accuracy: 0.8865 - val_loss: 0.0372 - val_accuracy: 0.9112 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0525 - accuracy: 0.8893 - val_loss: 0.0366 - val_accuracy: 0.9406 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0669 - accuracy: 0.8968 - val_loss: 0.0369 - val_accuracy: 0.9551 Epoch 9/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0743 - accuracy: 0.9008 - val_loss: 0.0390 - val_accuracy: 0.9480 Epoch 10/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0503 - accuracy: 0.9078 - val_loss: 0.0384 - val_accuracy: 0.9456 2938/2938 [==============================] - 1s 482us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3704 - accuracy: 0.6403 - val_loss: 0.0365 - val_accuracy: 0.9104 Epoch 2/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0573 - accuracy: 0.8636 - val_loss: 0.0361 - val_accuracy: 0.9319 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0591 - accuracy: 0.8747 - val_loss: 0.0358 - val_accuracy: 0.9499 Epoch 4/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0524 - accuracy: 0.9036 - val_loss: 0.0368 - val_accuracy: 0.9374 Epoch 5/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0538 - accuracy: 0.9004 - val_loss: 0.0357 - val_accuracy: 0.9457 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0524 - accuracy: 0.9029 - val_loss: 0.0367 - val_accuracy: 0.9404 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0540 - accuracy: 0.9029 - val_loss: 0.0370 - val_accuracy: 0.9480 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0589 - accuracy: 0.9025 - val_loss: 0.0380 - val_accuracy: 0.9523 Epoch 9/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0650 - accuracy: 0.8979 - val_loss: 0.0371 - val_accuracy: 0.9577 Epoch 10/10 5964/5964 [==============================] - 9s 1ms/step - loss: 0.0506 - accuracy: 0.9087 - val_loss: 0.0375 - val_accuracy: 0.9570 2938/2938 [==============================] - 1s 482us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3890 - accuracy: 0.6440 - val_loss: 0.0365 - val_accuracy: 0.9264 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0565 - accuracy: 0.8926 - val_loss: 0.0362 - val_accuracy: 0.9441 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0610 - accuracy: 0.9030 - val_loss: 0.0359 - val_accuracy: 0.9599 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.9153 - val_loss: 0.0372 - val_accuracy: 0.9500 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0548 - accuracy: 0.9074 - val_loss: 0.0361 - val_accuracy: 0.9541 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0536 - accuracy: 0.9081 - val_loss: 0.0375 - val_accuracy: 0.9440 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0527 - accuracy: 0.9068 - val_loss: 0.0366 - val_accuracy: 0.9602 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0708 - accuracy: 0.9062 - val_loss: 0.0369 - val_accuracy: 0.9635 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.9083 - val_loss: 0.0376 - val_accuracy: 0.9615 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9137 - val_loss: 0.0387 - val_accuracy: 0.9487 2938/2938 [==============================] - 2s 537us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3724 - accuracy: 0.6356 - val_loss: 0.0363 - val_accuracy: 0.9426 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0564 - accuracy: 0.8992 - val_loss: 0.0369 - val_accuracy: 0.9578 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0645 - accuracy: 0.9018 - val_loss: 0.0369 - val_accuracy: 0.9640 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0544 - accuracy: 0.9136 - val_loss: 0.0382 - val_accuracy: 0.9544 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.9108 - val_loss: 0.0364 - val_accuracy: 0.9679 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0535 - accuracy: 0.9121 - val_loss: 0.0371 - val_accuracy: 0.9533 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0524 - accuracy: 0.9113 - val_loss: 0.0370 - val_accuracy: 0.9595 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0713 - accuracy: 0.9055 - val_loss: 0.0369 - val_accuracy: 0.9658 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0754 - accuracy: 0.9138 - val_loss: 0.0378 - val_accuracy: 0.9631 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0499 - accuracy: 0.9124 - val_loss: 0.0372 - val_accuracy: 0.9607 2938/2938 [==============================] - 2s 510us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3878 - accuracy: 0.6345 - val_loss: 0.0369 - val_accuracy: 0.9202 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0572 - accuracy: 0.8875 - val_loss: 0.0371 - val_accuracy: 0.9387 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0621 - accuracy: 0.8984 - val_loss: 0.0366 - val_accuracy: 0.9496 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0557 - accuracy: 0.9120 - val_loss: 0.0379 - val_accuracy: 0.9516 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.9090 - val_loss: 0.0365 - val_accuracy: 0.9612 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0538 - accuracy: 0.9080 - val_loss: 0.0372 - val_accuracy: 0.9482 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9065 - val_loss: 0.0369 - val_accuracy: 0.9546 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0716 - accuracy: 0.9013 - val_loss: 0.0370 - val_accuracy: 0.9640 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0741 - accuracy: 0.9104 - val_loss: 0.0374 - val_accuracy: 0.9626 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9127 - val_loss: 0.0395 - val_accuracy: 0.9463 2938/2938 [==============================] - 2s 506us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3700 - accuracy: 0.6312 - val_loss: 0.0374 - val_accuracy: 0.8904 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.8438 - val_loss: 0.0362 - val_accuracy: 0.8682 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0559 - accuracy: 0.8597 - val_loss: 0.0354 - val_accuracy: 0.9190 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0538 - accuracy: 0.8852 - val_loss: 0.0363 - val_accuracy: 0.9333 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0566 - accuracy: 0.8913 - val_loss: 0.0354 - val_accuracy: 0.9499 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0525 - accuracy: 0.8981 - val_loss: 0.0361 - val_accuracy: 0.9404 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0531 - accuracy: 0.8991 - val_loss: 0.0362 - val_accuracy: 0.9511 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0659 - accuracy: 0.8971 - val_loss: 0.0368 - val_accuracy: 0.9555 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0691 - accuracy: 0.9051 - val_loss: 0.0372 - val_accuracy: 0.9546 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0502 - accuracy: 0.9095 - val_loss: 0.0378 - val_accuracy: 0.9558 2938/2938 [==============================] - 2s 517us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3858 - accuracy: 0.6415 - val_loss: 0.0365 - val_accuracy: 0.8895 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0552 - accuracy: 0.8550 - val_loss: 0.0357 - val_accuracy: 0.8995 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0558 - accuracy: 0.8622 - val_loss: 0.0358 - val_accuracy: 0.8978 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0536 - accuracy: 0.8653 - val_loss: 0.0363 - val_accuracy: 0.9072 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0541 - accuracy: 0.8723 - val_loss: 0.0357 - val_accuracy: 0.9082 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0534 - accuracy: 0.8799 - val_loss: 0.0359 - val_accuracy: 0.9071 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0537 - accuracy: 0.8739 - val_loss: 0.0374 - val_accuracy: 0.9171 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0575 - accuracy: 0.8798 - val_loss: 0.0365 - val_accuracy: 0.9453 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0607 - accuracy: 0.9001 - val_loss: 0.0371 - val_accuracy: 0.9421 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9025 - val_loss: 0.0374 - val_accuracy: 0.9420 2938/2938 [==============================] - 2s 508us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.4367 - accuracy: 0.6155 - val_loss: 0.0363 - val_accuracy: 0.9223 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0544 - accuracy: 0.8897 - val_loss: 0.0354 - val_accuracy: 0.9460 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0534 - accuracy: 0.9031 - val_loss: 0.0353 - val_accuracy: 0.9479 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0541 - accuracy: 0.9067 - val_loss: 0.0359 - val_accuracy: 0.9508 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.9073 - val_loss: 0.0356 - val_accuracy: 0.9597 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0529 - accuracy: 0.9133 - val_loss: 0.0357 - val_accuracy: 0.9563 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.9143 - val_loss: 0.0367 - val_accuracy: 0.9591 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0616 - accuracy: 0.9162 - val_loss: 0.0363 - val_accuracy: 0.9618 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0607 - accuracy: 0.9210 - val_loss: 0.0374 - val_accuracy: 0.9583 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0512 - accuracy: 0.9108 - val_loss: 0.0369 - val_accuracy: 0.9585 2938/2938 [==============================] - 2s 504us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3548 - accuracy: 0.6500 - val_loss: 0.0367 - val_accuracy: 0.9378 Epoch 2/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0565 - accuracy: 0.9002 - val_loss: 0.0369 - val_accuracy: 0.9587 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0657 - accuracy: 0.9036 - val_loss: 0.0371 - val_accuracy: 0.9642 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0552 - accuracy: 0.9156 - val_loss: 0.0377 - val_accuracy: 0.9593 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0555 - accuracy: 0.9130 - val_loss: 0.0365 - val_accuracy: 0.9689 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0535 - accuracy: 0.9129 - val_loss: 0.0370 - val_accuracy: 0.9566 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9115 - val_loss: 0.0367 - val_accuracy: 0.9597 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0712 - accuracy: 0.9032 - val_loss: 0.0368 - val_accuracy: 0.9647 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.9106 - val_loss: 0.0376 - val_accuracy: 0.9616 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9132 - val_loss: 0.0377 - val_accuracy: 0.9561 2938/2938 [==============================] - 2s 501us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3874 - accuracy: 0.6347 - val_loss: 0.0371 - val_accuracy: 0.9209 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0567 - accuracy: 0.8791 - val_loss: 0.0367 - val_accuracy: 0.9159 Epoch 3/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0582 - accuracy: 0.8769 - val_loss: 0.0361 - val_accuracy: 0.9112 Epoch 4/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.8828 - val_loss: 0.0382 - val_accuracy: 0.9209 Epoch 5/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.8901 - val_loss: 0.0363 - val_accuracy: 0.9506 Epoch 6/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0537 - accuracy: 0.9039 - val_loss: 0.0382 - val_accuracy: 0.9413 Epoch 7/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0527 - accuracy: 0.9042 - val_loss: 0.0375 - val_accuracy: 0.9553 Epoch 8/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0709 - accuracy: 0.9023 - val_loss: 0.0369 - val_accuracy: 0.9630 Epoch 9/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.9081 - val_loss: 0.0377 - val_accuracy: 0.9600 Epoch 10/10 5964/5964 [==============================] - 9s 2ms/step - loss: 0.0501 - accuracy: 0.9111 - val_loss: 0.0376 - val_accuracy: 0.9540 2938/2938 [==============================] - 2s 502us/step
Mean average precision over 10 runs: 0.0131
[0.0148, 0.0279, 0.0106, 0.007, 0.0117, 0.0159, 0.02, 0.0068, 0.0092, 0.0071]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0131 Coefficient of variation over 10 runs: 0.4922
[0.0148, 0.0279, 0.0106, 0.007, 0.0117, 0.0159, 0.02, 0.0068, 0.0092, 0.0071]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape)
for i in range(0,10):
# Call neural network API
model = Sequential()
# Generate hidden layer with 40 nodes using linear activation
model.add(Dense(units=40, activation='linear', \
activity_regularizer=regularizers.l1(10e-5), input_dim=29))
model.add(Dropout(0.05))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test_AE_noisy, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2963 - accuracy: 0.6827 - val_loss: 0.0062 - val_accuracy: 0.9705 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8486 - val_loss: 0.0049 - val_accuracy: 0.9806 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8456 - val_loss: 0.0053 - val_accuracy: 0.9807 Epoch 4/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0489 - accuracy: 0.8468 - val_loss: 0.0050 - val_accuracy: 0.9807 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0502 - accuracy: 0.8471 - val_loss: 0.0041 - val_accuracy: 0.9803 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0499 - accuracy: 0.8478 - val_loss: 0.0044 - val_accuracy: 0.9786 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8485 - val_loss: 0.0045 - val_accuracy: 0.9777 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0567 - accuracy: 0.8471 - val_loss: 0.0065 - val_accuracy: 0.9686 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0539 - accuracy: 0.8452 - val_loss: 0.0047 - val_accuracy: 0.9803 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8469 - val_loss: 0.0084 - val_accuracy: 0.9671 2938/2938 [==============================] - 2s 498us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3040 - accuracy: 0.6863 - val_loss: 0.0064 - val_accuracy: 0.9731 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0502 - accuracy: 0.8487 - val_loss: 0.0060 - val_accuracy: 0.9738 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8494 - val_loss: 0.0055 - val_accuracy: 0.9787 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0500 - accuracy: 0.8475 - val_loss: 0.0059 - val_accuracy: 0.9749 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8485 - val_loss: 0.0039 - val_accuracy: 0.9843 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8482 - val_loss: 0.0051 - val_accuracy: 0.9750 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8484 - val_loss: 0.0090 - val_accuracy: 0.9655 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0564 - accuracy: 0.8496 - val_loss: 0.0079 - val_accuracy: 0.9699 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0612 - accuracy: 0.8461 - val_loss: 0.0045 - val_accuracy: 0.9829 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8471 - val_loss: 0.0054 - val_accuracy: 0.9781 2938/2938 [==============================] - 2s 505us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.2983 - accuracy: 0.6837 - val_loss: 0.0054 - val_accuracy: 0.9781 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0504 - accuracy: 0.8485 - val_loss: 0.0047 - val_accuracy: 0.9787 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8487 - val_loss: 0.0046 - val_accuracy: 0.9824 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8493 - val_loss: 0.0067 - val_accuracy: 0.9716 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8478 - val_loss: 0.0041 - val_accuracy: 0.9822 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8507 - val_loss: 0.0048 - val_accuracy: 0.9758 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8496 - val_loss: 0.0047 - val_accuracy: 0.9796 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0532 - accuracy: 0.8465 - val_loss: 0.0062 - val_accuracy: 0.9725 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0606 - accuracy: 0.8474 - val_loss: 0.0054 - val_accuracy: 0.9762 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0483 - accuracy: 0.8491 - val_loss: 0.0050 - val_accuracy: 0.9782 2938/2938 [==============================] - 2s 496us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3338 - accuracy: 0.6718 - val_loss: 0.0058 - val_accuracy: 0.9746 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8503 - val_loss: 0.0055 - val_accuracy: 0.9764 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8476 - val_loss: 0.0042 - val_accuracy: 0.9841 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8474 - val_loss: 0.0058 - val_accuracy: 0.9726 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0508 - accuracy: 0.8491 - val_loss: 0.0042 - val_accuracy: 0.9829 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8474 - val_loss: 0.0055 - val_accuracy: 0.9700 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8476 - val_loss: 0.0042 - val_accuracy: 0.9810 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0540 - accuracy: 0.8457 - val_loss: 0.0066 - val_accuracy: 0.9672 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0582 - accuracy: 0.8461 - val_loss: 0.0046 - val_accuracy: 0.9815 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8470 - val_loss: 0.0077 - val_accuracy: 0.9732 2938/2938 [==============================] - 2s 507us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3136 - accuracy: 0.6768 - val_loss: 0.0055 - val_accuracy: 0.9767 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8476 - val_loss: 0.0043 - val_accuracy: 0.9822 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0503 - accuracy: 0.8481 - val_loss: 0.0050 - val_accuracy: 0.9787 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0486 - accuracy: 0.8493 - val_loss: 0.0072 - val_accuracy: 0.9763 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8493 - val_loss: 0.0046 - val_accuracy: 0.9776 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8478 - val_loss: 0.0043 - val_accuracy: 0.9819 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8500 - val_loss: 0.0061 - val_accuracy: 0.9735 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0616 - accuracy: 0.8471 - val_loss: 0.0049 - val_accuracy: 0.9761 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0587 - accuracy: 0.8490 - val_loss: 0.0064 - val_accuracy: 0.9750 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8490 - val_loss: 0.0050 - val_accuracy: 0.9806 2938/2938 [==============================] - 2s 501us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3261 - accuracy: 0.6746 - val_loss: 0.0059 - val_accuracy: 0.9734 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0514 - accuracy: 0.8482 - val_loss: 0.0044 - val_accuracy: 0.9813 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8494 - val_loss: 0.0045 - val_accuracy: 0.9817 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8488 - val_loss: 0.0052 - val_accuracy: 0.9743 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0494 - accuracy: 0.8496 - val_loss: 0.0039 - val_accuracy: 0.9844 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8498 - val_loss: 0.0059 - val_accuracy: 0.9753 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8486 - val_loss: 0.0046 - val_accuracy: 0.9805 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0678 - accuracy: 0.8490 - val_loss: 0.0062 - val_accuracy: 0.9726 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0554 - accuracy: 0.8472 - val_loss: 0.0040 - val_accuracy: 0.9838 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0498 - accuracy: 0.8472 - val_loss: 0.0058 - val_accuracy: 0.9763 2938/2938 [==============================] - 2s 510us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.3120 - accuracy: 0.6737 - val_loss: 0.0055 - val_accuracy: 0.9749 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0512 - accuracy: 0.8498 - val_loss: 0.0044 - val_accuracy: 0.9818 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8491 - val_loss: 0.0043 - val_accuracy: 0.9812 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8498 - val_loss: 0.0069 - val_accuracy: 0.9743 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0497 - accuracy: 0.8485 - val_loss: 0.0043 - val_accuracy: 0.9814 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0498 - accuracy: 0.8495 - val_loss: 0.0048 - val_accuracy: 0.9757 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0489 - accuracy: 0.8485 - val_loss: 0.0044 - val_accuracy: 0.9789 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0572 - accuracy: 0.8478 - val_loss: 0.0055 - val_accuracy: 0.9702 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0543 - accuracy: 0.8465 - val_loss: 0.0040 - val_accuracy: 0.9823 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8490 - val_loss: 0.0054 - val_accuracy: 0.9795 2938/2938 [==============================] - 2s 506us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.3053 - accuracy: 0.6925 - val_loss: 0.0054 - val_accuracy: 0.9751 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0494 - accuracy: 0.8490 - val_loss: 0.0047 - val_accuracy: 0.9802 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8483 - val_loss: 0.0044 - val_accuracy: 0.9819 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8471 - val_loss: 0.0056 - val_accuracy: 0.9782 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0519 - accuracy: 0.8466 - val_loss: 0.0038 - val_accuracy: 0.9854 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8516 - val_loss: 0.0046 - val_accuracy: 0.9781 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0487 - accuracy: 0.8492 - val_loss: 0.0055 - val_accuracy: 0.9777 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0575 - accuracy: 0.8501 - val_loss: 0.0058 - val_accuracy: 0.9734 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0533 - accuracy: 0.8479 - val_loss: 0.0059 - val_accuracy: 0.9764 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0491 - accuracy: 0.8489 - val_loss: 0.0056 - val_accuracy: 0.9786 2938/2938 [==============================] - 2s 509us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.2979 - accuracy: 0.6819 - val_loss: 0.0058 - val_accuracy: 0.9726 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0512 - accuracy: 0.8480 - val_loss: 0.0047 - val_accuracy: 0.9785 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8479 - val_loss: 0.0048 - val_accuracy: 0.9809 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8472 - val_loss: 0.0046 - val_accuracy: 0.9799 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8493 - val_loss: 0.0041 - val_accuracy: 0.9809 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8489 - val_loss: 0.0043 - val_accuracy: 0.9799 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8460 - val_loss: 0.0049 - val_accuracy: 0.9774 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0622 - accuracy: 0.8471 - val_loss: 0.0058 - val_accuracy: 0.9731 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0513 - accuracy: 0.8476 - val_loss: 0.0047 - val_accuracy: 0.9777 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8480 - val_loss: 0.0074 - val_accuracy: 0.9745 2938/2938 [==============================] - 2s 495us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.3105 - accuracy: 0.6789 - val_loss: 0.0058 - val_accuracy: 0.9769 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0511 - accuracy: 0.8462 - val_loss: 0.0041 - val_accuracy: 0.9823 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8490 - val_loss: 0.0041 - val_accuracy: 0.9837 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8497 - val_loss: 0.0066 - val_accuracy: 0.9733 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8488 - val_loss: 0.0038 - val_accuracy: 0.9819 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.0498 - accuracy: 0.8486 - val_loss: 0.0047 - val_accuracy: 0.9784 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8517 - val_loss: 0.0048 - val_accuracy: 0.9786 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0550 - accuracy: 0.8464 - val_loss: 0.0062 - val_accuracy: 0.9722 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0563 - accuracy: 0.8481 - val_loss: 0.0040 - val_accuracy: 0.9835 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8478 - val_loss: 0.0078 - val_accuracy: 0.9733 2938/2938 [==============================] - 2s 501us/step
Mean average precision over 10 runs: 0.0239
[0.0155, 0.0307, 0.0122, 0.025, 0.0202, 0.02, 0.0257, 0.0244, 0.0343, 0.0308]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0239 Coefficient of variation over 10 runs: 0.2791
[0.0155, 0.0307, 0.0122, 0.025, 0.0202, 0.02, 0.0257, 0.0244, 0.0343, 0.0308]
tf.random.set_seed(42)
np.random.seed(42)
# 10 runs - We will capture mean of average precision
test_scores = []
noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape)
for i in range(0,10):
# Call neural network API
model = Sequential()
# Generate hidden layer with 40 nodes using linear activation
model.add(Dense(units=40, activation='relu', \
activity_regularizer=regularizers.l1(10e-5), input_dim=29))
model.add(Dropout(0.05))
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
# Compile the model
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# Train the model
num_epochs = 10
batch_size = 32
history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
epochs=num_epochs,
batch_size=batch_size,
shuffle=True,
validation_data=(X_train_AE, X_train_AE),
verbose=1)
# Evaluate on test set
predictions = model.predict(X_test_AE_noisy, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
test_scores.append(avgPrecision)
model.reset_states()
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.4365 - accuracy: 0.5372 - val_loss: 0.0228 - val_accuracy: 0.9172 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1360 - accuracy: 0.7204 - val_loss: 0.0184 - val_accuracy: 0.9130 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1267 - accuracy: 0.7291 - val_loss: 0.0152 - val_accuracy: 0.9225 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1177 - accuracy: 0.7337 - val_loss: 0.0155 - val_accuracy: 0.9174 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1180 - accuracy: 0.7330 - val_loss: 0.0128 - val_accuracy: 0.9232 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1175 - accuracy: 0.7349 - val_loss: 0.0135 - val_accuracy: 0.9229 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1147 - accuracy: 0.7363 - val_loss: 0.0128 - val_accuracy: 0.9234 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1165 - accuracy: 0.7366 - val_loss: 0.0141 - val_accuracy: 0.9201 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1158 - accuracy: 0.7390 - val_loss: 0.0125 - val_accuracy: 0.9236 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1092 - accuracy: 0.7387 - val_loss: 0.0141 - val_accuracy: 0.9229 2938/2938 [==============================] - 2s 499us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.4213 - accuracy: 0.5432 - val_loss: 0.0220 - val_accuracy: 0.9098 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1317 - accuracy: 0.7235 - val_loss: 0.0186 - val_accuracy: 0.9203 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1282 - accuracy: 0.7284 - val_loss: 0.0154 - val_accuracy: 0.9275 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1204 - accuracy: 0.7328 - val_loss: 0.0159 - val_accuracy: 0.9176 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1181 - accuracy: 0.7377 - val_loss: 0.0126 - val_accuracy: 0.9219 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1171 - accuracy: 0.7404 - val_loss: 0.0147 - val_accuracy: 0.9179 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1166 - accuracy: 0.7399 - val_loss: 0.0127 - val_accuracy: 0.9229 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1165 - accuracy: 0.7415 - val_loss: 0.0126 - val_accuracy: 0.9237 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1143 - accuracy: 0.7403 - val_loss: 0.0127 - val_accuracy: 0.9255 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1142 - accuracy: 0.7433 - val_loss: 0.0149 - val_accuracy: 0.9240 2938/2938 [==============================] - 2s 495us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4398 - accuracy: 0.5334 - val_loss: 0.0231 - val_accuracy: 0.9095 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1333 - accuracy: 0.7188 - val_loss: 0.0177 - val_accuracy: 0.9130 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1271 - accuracy: 0.7283 - val_loss: 0.0145 - val_accuracy: 0.9233 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1219 - accuracy: 0.7308 - val_loss: 0.0171 - val_accuracy: 0.9126 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1215 - accuracy: 0.7351 - val_loss: 0.0131 - val_accuracy: 0.9226 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1133 - accuracy: 0.7375 - val_loss: 0.0128 - val_accuracy: 0.9224 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1126 - accuracy: 0.7370 - val_loss: 0.0128 - val_accuracy: 0.9243 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1117 - accuracy: 0.7381 - val_loss: 0.0147 - val_accuracy: 0.9177 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1236 - accuracy: 0.7340 - val_loss: 0.0127 - val_accuracy: 0.9268 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1107 - accuracy: 0.7388 - val_loss: 0.0137 - val_accuracy: 0.9253 2938/2938 [==============================] - 2s 548us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4654 - accuracy: 0.5268 - val_loss: 0.0241 - val_accuracy: 0.9015 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1316 - accuracy: 0.7230 - val_loss: 0.0168 - val_accuracy: 0.9214 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1252 - accuracy: 0.7312 - val_loss: 0.0143 - val_accuracy: 0.9240 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1198 - accuracy: 0.7346 - val_loss: 0.0152 - val_accuracy: 0.9199 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1182 - accuracy: 0.7351 - val_loss: 0.0124 - val_accuracy: 0.9256 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1150 - accuracy: 0.7383 - val_loss: 0.0137 - val_accuracy: 0.9140 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1138 - accuracy: 0.7373 - val_loss: 0.0135 - val_accuracy: 0.9192 Epoch 8/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1230 - accuracy: 0.7372 - val_loss: 0.0140 - val_accuracy: 0.9219 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1156 - accuracy: 0.7355 - val_loss: 0.0141 - val_accuracy: 0.9177 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1102 - accuracy: 0.7374 - val_loss: 0.0154 - val_accuracy: 0.9208 2938/2938 [==============================] - 2s 498us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4246 - accuracy: 0.5415 - val_loss: 0.0222 - val_accuracy: 0.9146 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1308 - accuracy: 0.7255 - val_loss: 0.0178 - val_accuracy: 0.9193 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1229 - accuracy: 0.7305 - val_loss: 0.0134 - val_accuracy: 0.9281 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1172 - accuracy: 0.7352 - val_loss: 0.0155 - val_accuracy: 0.9184 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1176 - accuracy: 0.7391 - val_loss: 0.0123 - val_accuracy: 0.9260 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1116 - accuracy: 0.7433 - val_loss: 0.0124 - val_accuracy: 0.9214 Epoch 7/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1121 - accuracy: 0.7393 - val_loss: 0.0152 - val_accuracy: 0.9163 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1177 - accuracy: 0.7389 - val_loss: 0.0140 - val_accuracy: 0.9178 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1227 - accuracy: 0.7377 - val_loss: 0.0124 - val_accuracy: 0.9218 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1111 - accuracy: 0.7418 - val_loss: 0.0134 - val_accuracy: 0.9226 2938/2938 [==============================] - 2s 507us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4539 - accuracy: 0.5296 - val_loss: 0.0256 - val_accuracy: 0.9088 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1347 - accuracy: 0.7207 - val_loss: 0.0179 - val_accuracy: 0.9229 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1274 - accuracy: 0.7280 - val_loss: 0.0158 - val_accuracy: 0.9200 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1189 - accuracy: 0.7341 - val_loss: 0.0162 - val_accuracy: 0.9179 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1166 - accuracy: 0.7351 - val_loss: 0.0126 - val_accuracy: 0.9236 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1148 - accuracy: 0.7355 - val_loss: 0.0135 - val_accuracy: 0.9213 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1154 - accuracy: 0.7379 - val_loss: 0.0129 - val_accuracy: 0.9192 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1228 - accuracy: 0.7344 - val_loss: 0.0137 - val_accuracy: 0.9148 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1364 - accuracy: 0.7366 - val_loss: 0.0122 - val_accuracy: 0.9190 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1100 - accuracy: 0.7373 - val_loss: 0.0133 - val_accuracy: 0.9181 2938/2938 [==============================] - 1s 492us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4422 - accuracy: 0.5274 - val_loss: 0.0229 - val_accuracy: 0.9164 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1381 - accuracy: 0.7222 - val_loss: 0.0196 - val_accuracy: 0.9217 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1273 - accuracy: 0.7277 - val_loss: 0.0159 - val_accuracy: 0.9234 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1205 - accuracy: 0.7330 - val_loss: 0.0157 - val_accuracy: 0.9190 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1195 - accuracy: 0.7345 - val_loss: 0.0137 - val_accuracy: 0.9241 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1149 - accuracy: 0.7370 - val_loss: 0.0137 - val_accuracy: 0.9221 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1141 - accuracy: 0.7380 - val_loss: 0.0130 - val_accuracy: 0.9220 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1130 - accuracy: 0.7382 - val_loss: 0.0136 - val_accuracy: 0.9168 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1128 - accuracy: 0.7384 - val_loss: 0.0122 - val_accuracy: 0.9205 Epoch 10/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1126 - accuracy: 0.7381 - val_loss: 0.0135 - val_accuracy: 0.9248 2938/2938 [==============================] - 2s 559us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4188 - accuracy: 0.5494 - val_loss: 0.0238 - val_accuracy: 0.9029 Epoch 2/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1334 - accuracy: 0.7220 - val_loss: 0.0183 - val_accuracy: 0.9119 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1262 - accuracy: 0.7252 - val_loss: 0.0162 - val_accuracy: 0.9178 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1202 - accuracy: 0.7299 - val_loss: 0.0153 - val_accuracy: 0.9206 Epoch 5/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1191 - accuracy: 0.7333 - val_loss: 0.0136 - val_accuracy: 0.9242 Epoch 6/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1144 - accuracy: 0.7366 - val_loss: 0.0157 - val_accuracy: 0.9065 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1148 - accuracy: 0.7373 - val_loss: 0.0129 - val_accuracy: 0.9259 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1248 - accuracy: 0.7377 - val_loss: 0.0161 - val_accuracy: 0.9140 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1130 - accuracy: 0.7385 - val_loss: 0.0116 - val_accuracy: 0.9246 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1136 - accuracy: 0.7379 - val_loss: 0.0128 - val_accuracy: 0.9265 2938/2938 [==============================] - 2s 551us/step
Epoch 1/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.4304 - accuracy: 0.5351 - val_loss: 0.0231 - val_accuracy: 0.9057 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1340 - accuracy: 0.7215 - val_loss: 0.0176 - val_accuracy: 0.9148 Epoch 3/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1254 - accuracy: 0.7280 - val_loss: 0.0157 - val_accuracy: 0.9158 Epoch 4/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1206 - accuracy: 0.7312 - val_loss: 0.0143 - val_accuracy: 0.9264 Epoch 5/10 5964/5964 [==============================] - 12s 2ms/step - loss: 0.1168 - accuracy: 0.7337 - val_loss: 0.0124 - val_accuracy: 0.9248 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1131 - accuracy: 0.7398 - val_loss: 0.0135 - val_accuracy: 0.9214 Epoch 7/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1151 - accuracy: 0.7389 - val_loss: 0.0124 - val_accuracy: 0.9268 Epoch 8/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1148 - accuracy: 0.7407 - val_loss: 0.0138 - val_accuracy: 0.9230 Epoch 9/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1145 - accuracy: 0.7422 - val_loss: 0.0126 - val_accuracy: 0.9245 Epoch 10/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1105 - accuracy: 0.7425 - val_loss: 0.0150 - val_accuracy: 0.9243 2938/2938 [==============================] - 2s 562us/step
Epoch 1/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.4332 - accuracy: 0.5407 - val_loss: 0.0232 - val_accuracy: 0.9062 Epoch 2/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1355 - accuracy: 0.7184 - val_loss: 0.0182 - val_accuracy: 0.9128 Epoch 3/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1269 - accuracy: 0.7259 - val_loss: 0.0147 - val_accuracy: 0.9247 Epoch 4/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1200 - accuracy: 0.7315 - val_loss: 0.0150 - val_accuracy: 0.9249 Epoch 5/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1181 - accuracy: 0.7363 - val_loss: 0.0124 - val_accuracy: 0.9257 Epoch 6/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1141 - accuracy: 0.7394 - val_loss: 0.0153 - val_accuracy: 0.9120 Epoch 7/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1125 - accuracy: 0.7406 - val_loss: 0.0127 - val_accuracy: 0.9282 Epoch 8/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1279 - accuracy: 0.7382 - val_loss: 0.0151 - val_accuracy: 0.9163 Epoch 9/10 5964/5964 [==============================] - 11s 2ms/step - loss: 0.1172 - accuracy: 0.7421 - val_loss: 0.0124 - val_accuracy: 0.9215 Epoch 10/10 5964/5964 [==============================] - 10s 2ms/step - loss: 0.1096 - accuracy: 0.7419 - val_loss: 0.0160 - val_accuracy: 0.9158 2938/2938 [==============================] - 2s 515us/step
Mean average precision over 10 runs: 0.0417
[0.0301, 0.0311, 0.0495, 0.0254, 0.0294, 0.0546, 0.0368, 0.0234, 0.0366, 0.1002]
print(f'Mean average precision over 10 runs: {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs: 0.0417 Coefficient of variation over 10 runs: 0.52
[0.0301, 0.0311, 0.0495, 0.0254, 0.0294, 0.0546, 0.0368, 0.0234, 0.0366, 0.1002]
import datetime as dt
print("Completed: ", dt.datetime.now())
Completed: 2021-05-06 15:56:19.284316